deploytool/cmd/luci_deploy/deploy.go - external/github.com/luci/luci-go.git - Git at Google

 // Copyright 2016 The LUCI Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package main

 import (
 	"sort"

 	"github.com/luci/luci-go/common/cli"
 	"github.com/luci/luci-go/common/errors"
 	"github.com/luci/luci-go/common/flag/flagenum"
 	log "github.com/luci/luci-go/common/logging"
 	"github.com/luci/luci-go/deploytool/managedfs"

 	"github.com/maruel/subcommands"
 )

 type deployStage int

 const (
 	// deployStaging is the staging stage, where local filesystem and build
 	// artifacts are constructed. Build scripts are executed, and generated files
 	// and directories are created. This will leave the user with a directory that
 	// can be used to build from.
 	deployStaging deployStage = iota
 	// deployLocalBuild performs a local build of the components, offering both a
 	// sanity check prior to engaging remote services and creating build artifacts
 	// ready to push remotely.
 	deployLocalBuild deployStage = iota
 	// deployPush pushes local build artifacts to the remote service, but doesn't
 	// enable them.
 	deployPush deployStage = iota
 	// deployCommit commits the pushed artifacts to the remote service, activating
 	// them.
 	deployCommit deployStage = iota
 )

 var deployStageFlagEnum = flagenum.Enum{
 	"stage":      deployStaging,
 	"localbuild": deployLocalBuild,
 	"push":       deployPush,
 	"commit":     deployCommit,
 }

 func (s *deployStage) Set(v string) error { return deployStageFlagEnum.FlagSet(s, v) }
 func (s deployStage) String() string      { return deployStageFlagEnum.FlagString(s) }

 type deployParams struct {
 	// stage is the highest deployment stage to deploy through.
 	stage deployStage

 	// ignoreCurrentVersion, if true, instructs depoyment logic to proceed with
 	// deployment even if the currently-deployed version matches the current
 	// source version.
 	ignoreCurrentVersion bool
 	// commitTainted, if true, instructs the deploy tool to perform commit
 	// operations on tainted Components.
 	commitTainted bool
 	// forceVersion forces the use of the supplied version instead of the one
 	// generated by the source.
 	forceVersion cloudProjectVersion
 }

 var cmdDeploy = subcommands.Command{
 	UsageLine: "deploy [DEPLOYMENT.[.COMPONENT]]...",
 	ShortDesc: "Deploys some or all of a deployment.",
 	LongDesc:  "Deploys a Deployment. If specific projects are named, only those projects will be deployed.",
 	CommandRun: func() subcommands.CommandRun {
 		var cmd cmdDeployRun
 		cmd.dp.params.stage = deployCommit

 		fs := cmd.GetFlags()
 		fs.BoolVar(&cmd.checkout, "checkout", false,
 			"Refresh the checkout prior to deployment.")
 		fs.Var(&cmd.dp.params.stage, "stage",
 			"Run the deployment sequence up through this stage ["+deployStageFlagEnum.Choices()+"].")
 		fs.BoolVar(&cmd.dp.params.commitTainted, "commit-tainted", false,
 			"Install new deployments even if their source is tained.")
 		fs.BoolVar(&cmd.dp.params.ignoreCurrentVersion, "ignore-current-version", false,
 			"Push/install new deployments even if the currently-deployed version would not change or could not be parsed.")
 		fs.StringVar(&cmd.manualVersion, "version", "",
 			"Manually specify the version to use for deployment. Implies tainted.")
 		return &cmd
 	},
 }

 type cmdDeployRun struct {
 	subcommands.CommandRunBase

 	// checkout, if true, refreshes the checkout prior to deployment.
 	checkout bool
 	// manualVersion is the manually-specified cloud project version.
 	manualVersion string

 	dp deploymentPlan
 }

 func (cmd *cmdDeployRun) Run(app subcommands.Application, args []string, env subcommands.Env) int {
 	a, c := app.(*application), cli.GetContext(app, cmd, env)

 	// If a manual version was specified, use it.
 	if cmd.manualVersion != "" {
 		cmd.dp.params.forceVersion = makeStringCloudProjectVersion(cmd.manualVersion)
 	}

 	err := a.runWork(c, func(w *work) error {
 		// Perform our planned checkout.
 		if cmd.checkout {
 			// Perform a full checkout.
 			if err := checkout(w, &a.layout, false); err != nil {
 				return errors.Annotate(err, "failed to checkout sources").Err()
 			}
 		}

 		// Load our frozen checkout.
 		if err := a.layout.loadFrozenLayout(w); err != nil {
 			return errors.Annotate(err, "failed to load frozen checkout").Err()
 		}
 		return nil
 	})
 	if err != nil {
 		logError(c, err, "Failed to perform checkout.")
 		return 1
 	}

 	// Figure out what we're going to deploy.
 	for _, arg := range args {
 		matched := false
 		err := a.layout.matchDeploymentComponent(arg, func(dep *layoutDeployment, comp *layoutDeploymentComponent) {
 			matched = true

 			if comp != nil {
 				cmd.dp.addProjectComponent(comp)
 			} else {
 				cmd.dp.addDeployment(dep)
 			}
 		})
 		if err != nil {
 			logError(c, err, "Failed during matching.")
 			return 1
 		}
 		if !matched {
 			log.Fields{
 				"target": arg,
 			}.Errorf(c, "Deploy target did not match any configured deployments or components.")
 			return 1
 		}
 	}

 	err = a.runWork(c, func(w *work) error {
 		if err := cmd.dp.initialize(w, &a.layout); err != nil {
 			return errors.Annotate(err, "failed to initialize").Err()
 		}

 		return cmd.dp.deploy(w)
 	})
 	if err != nil {
 		logError(c, err, "Failed to perform deployment.")
 		return 1
 	}
 	return 0
 }

 type deploymentPlan struct {
 	// comps maps DEPLOYMENT/PROJECT/COMPONENT to its specific deployable project
 	// component.
 	comps map[string]*layoutDeploymentComponent

 	// deployments tracks which deployments we've specified.
 	deployments map[title]struct{}

 	// params are the deployment parameters to forward to deployment operations.
 	params deployParams

 	// reg is the registry of deployable Components.
 	reg deploymentRegistry

 	// layout is the deployment layout. It is initialized during "initialize".
 	layout *deployLayout
 }

 func (dp *deploymentPlan) addProjectComponent(comp *layoutDeploymentComponent) {
 	// Mark this specific component.
 	if dp.comps == nil {
 		dp.comps = make(map[string]*layoutDeploymentComponent)
 	}
 	dp.comps[comp.String()] = comp

 	// Mark that we've used this deployment.
 	if dp.deployments == nil {
 		dp.deployments = make(map[title]struct{})
 	}
 	dp.deployments[comp.dep.title] = struct{}{}
 }

 func (dp *deploymentPlan) addDeployment(dep *layoutDeployment) {
 	for _, comp := range dep.components {
 		dp.addProjectComponent(comp)
 	}
 }

 func (dp *deploymentPlan) deployedComponents() []string {
 	comps := make([]string, 0, len(dp.comps))
 	for path := range dp.comps {
 		comps = append(comps, path)
 	}
 	sort.Strings(comps)
 	return comps
 }

 func (dp *deploymentPlan) initialize(w *work, l *deployLayout) error {
 	dp.layout = l

 	// Load all deployable Components from checkout.
 	for _, depName := range l.deploymentNames {
 		dep := l.deployments[depName]

 		if _, ok := dp.deployments[dep.title]; !ok {
 			continue
 		}

 		// Load the application components. We will iterate through them in the
 		// order in which they are defined in the Application's protobuf.
 		for _, compName := range dep.componentNames {
 			comp := dep.components[compName]

 			// Only pass the registrar if this component is being deployed.
 			var pReg componentRegistrar
 			if _, ok := dp.comps[comp.String()]; ok {
 				pReg = &dp.reg
 			}
 			if err := comp.loadSourceComponent(pReg); err != nil {
 				return errors.Annotate(err, "failed to load component %q", comp).Err()
 			}
 		}
 	}

 	return nil
 }

 func (dp *deploymentPlan) deploy(w *work) error {
 	// Create our working root and staging/build subdirectories.
 	fs, err := dp.layout.workingFilesystem()
 	if err != nil {
 		return errors.Annotate(err, "failed to create working directory").Err()
 	}

 	stagingDir, err := fs.Base().EnsureDirectory("staging")
 	if err != nil {
 		return errors.Annotate(err, "failed to create staging directory").Err()
 	}

 	// Stage: Staging
 	if err := dp.reg.stage(w, stagingDir, &dp.params); err != nil {
 		return errors.Annotate(err, "failed to stage").Err()
 	}
 	if dp.params.stage <= deployStaging {
 		return nil
 	}

 	// Stage: Local Build
 	if err := dp.reg.localBuild(w); err != nil {
 		return errors.Annotate(err, "failed to perform local build").Err()
 	}
 	if dp.params.stage <= deployLocalBuild {
 		return nil
 	}

 	// Stage: Push
 	if err := dp.reg.push(w); err != nil {
 		return errors.Annotate(err, "failed to push components").Err()
 	}
 	if dp.params.stage <= deployPush {
 		return nil
 	}

 	// Stage: Commit
 	if err := dp.reg.commit(w); err != nil {
 		return errors.Annotate(err, "failed to commit").Err()
 	}
 	return nil
 }

 // deploymentRegistry is the registry of all prepared deployments.
 //
 // While deployment is specified and staged at a component level, actual
 // deployment is allowed to happen at a practical level (e.g., a single
 // cloud project's AppEngine configuration, a single Google Container Engine
 // cluster, etc.).
 //
 // This will track individual components, as well as integrate them into larger
 // action plans.
 type deploymentRegistry struct {
 	// components is a map of components that have been deployed.
 	components map[string]*layoutDeploymentComponent
 	// componentNames is the sorted list of component names in components.
 	componentNames []string

 	// gaeProjects maps Google AppEngine deployment state to specific GAE
 	// projects.
 	gaeProjects map[string]*gaeDeployment
 	// gaeProjectNames is the sorted list of registered GAE project names.
 	gaeProjectNames []string

 	// gkeProjects is the set of Google Container Engine deployments for a given
 	// cloud project.
 	gkeProjects map[string]*containerEngineDeployment
 	// gkeProjectNames is the sorted list of registered GKE project names.
 	gkeProjectNames []string
 }

 func (reg *deploymentRegistry) addComponent(comp *layoutDeploymentComponent) {
 	name := comp.comp.Name
 	if _, has := reg.components[name]; has {
 		return
 	}

 	if reg.components == nil {
 		reg.components = make(map[string]*layoutDeploymentComponent)
 	}
 	reg.components[name] = comp
 	reg.componentNames = append(reg.componentNames, name)
 }

 func (reg *deploymentRegistry) addGAEModule(module *layoutDeploymentGAEModule) {
 	// Get/create the AppEngine project for this module.
 	cloudProjectName := module.comp.dep.cloudProject.Name
 	gaeD := reg.gaeProjects[cloudProjectName]
 	if gaeD == nil {
 		gaeD = makeGAEDeployment(module.comp.dep.cloudProject)

 		if reg.gaeProjects == nil {
 			reg.gaeProjects = make(map[string]*gaeDeployment)
 		}
 		reg.gaeProjects[cloudProjectName] = gaeD
 		reg.gaeProjectNames = append(reg.gaeProjectNames, cloudProjectName)
 	}

 	gaeD.addModule(module)
 	reg.addComponent(module.comp)
 }

 func (reg *deploymentRegistry) addGKEPod(pb *layoutDeploymentGKEPodBinding) {
 	// Get/create the AppEngine project for this module.
 	cloudProjectName := pb.cluster.cloudProject.Name
 	gkeD := reg.gkeProjects[cloudProjectName]
 	if gkeD == nil {
 		gkeD = &containerEngineDeployment{
 			project:  pb.cluster.cloudProject,
 			clusters: make(map[string]*containerEngineDeploymentCluster),
 		}

 		if reg.gkeProjects == nil {
 			reg.gkeProjects = make(map[string]*containerEngineDeployment)
 		}
 		reg.gkeProjects[cloudProjectName] = gkeD
 		reg.gkeProjectNames = append(reg.gkeProjectNames, cloudProjectName)
 	}

 	// Register this pod in its cluster.
 	gkeD.addCluster(pb.cluster).attachPod(pb)
 }

 // appEngineModulesOnly clears all deployment parameters that are not AppEngine
 // modules.
 func (reg *deploymentRegistry) appEngineModulesOnly() {
 	reg.gkeProjects, reg.gkeProjectNames = nil, nil
 }

 // stage performs staging, preparing deployment structures on the local
 // filesystem.
 func (reg *deploymentRegistry) stage(w *work, stageDir *managedfs.Dir, params *deployParams) error {
 	// Sort our name lists.
 	sort.Strings(reg.componentNames)
 	sort.Strings(reg.gaeProjectNames)
 	sort.Strings(reg.gkeProjectNames)

 	// All components can be independently staged in parallel.
 	//
 	// We will still enumerate over them in sorted order for determinism.
 	return w.RunMulti(func(workC chan<- func() error) {
 		// AppEngine Projects
 		for _, name := range reg.gaeProjectNames {
 			proj := reg.gaeProjects[name]
 			workC <- func() error {
 				depDir, err := stageDir.EnsureDirectory(string(proj.project.dep.title), "appengine")
 				if err != nil {
 					return errors.Annotate(err, "failed to create deployment directory for %q", proj.project.dep.title).Err()
 				}
 				return proj.stage(w, depDir, params)
 			}
 		}

 		// Container Engine Projects
 		for _, name := range reg.gkeProjectNames {
 			proj := reg.gkeProjects[name]
 			workC <- func() error {
 				depDir, err := stageDir.EnsureDirectory(string(proj.project.dep.title), "container_engine")
 				if err != nil {
 					return errors.Annotate(err, "failed to create deployment directory for %q", proj.project.dep.title).Err()
 				}
 				return proj.stage(w, depDir, params)
 			}
 		}
 	})
 }

 // build performs the build stage on staged components.
 func (reg *deploymentRegistry) localBuild(w *work) error {
 	// Enumerate over them in sorted order for determinism.
 	return w.RunMulti(func(workC chan<- func() error) {
 		// AppEngine Projects
 		for _, name := range reg.gaeProjectNames {
 			proj := reg.gaeProjects[name]
 			workC <- func() error {
 				return proj.localBuild(w)
 			}
 		}

 		// Container Engine Projects
 		for _, name := range reg.gkeProjectNames {
 			proj := reg.gkeProjects[name]
 			workC <- func() error {
 				return proj.localBuild(w)
 			}
 		}
 	})
 }

 // push builds and pushes the deployments to remote services, but does not
 // commit to them.
 func (reg *deploymentRegistry) push(w *work) error {
 	// All components can be independently pushed in parallel.
 	//
 	// We will still enumerate over them in sorted order for determinism.
 	return w.RunMulti(func(workC chan<- func() error) {
 		// AppEngine Projects
 		for _, name := range reg.gaeProjectNames {
 			proj := reg.gaeProjects[name]
 			workC <- func() error {
 				return proj.push(w)
 			}
 		}

 		// Container Engine Projects
 		for _, name := range reg.gkeProjectNames {
 			proj := reg.gkeProjects[name]
 			workC <- func() error {
 				return proj.push(w)
 			}
 		}
 	})
 }

 // commit commits the deployment to the remote services.
 func (reg *deploymentRegistry) commit(w *work) error {
 	// All components can be independently committed in parallel.
 	//
 	// We will still enumerate over them in sorted order for determinism.
 	return w.RunMulti(func(workC chan<- func() error) {
 		// AppEngine Projects
 		for _, name := range reg.gaeProjectNames {
 			proj := reg.gaeProjects[name]
 			workC <- func() error {
 				return proj.commit(w)
 			}
 		}

 		// Container Engine Projects
 		for _, name := range reg.gkeProjectNames {
 			proj := reg.gkeProjects[name]
 			workC <- func() error {
 				return proj.commit(w)
 			}
 		}
 	})
 }
	// Copyright 2016 The LUCI Authors.
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package main

	import (
	"sort"

	"github.com/luci/luci-go/common/cli"
	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/flag/flagenum"
	log "github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/deploytool/managedfs"

	"github.com/maruel/subcommands"
	)

	type deployStage int

	const (
	// deployStaging is the staging stage, where local filesystem and build
	// artifacts are constructed. Build scripts are executed, and generated files
	// and directories are created. This will leave the user with a directory that
	// can be used to build from.
	deployStaging deployStage = iota
	// deployLocalBuild performs a local build of the components, offering both a
	// sanity check prior to engaging remote services and creating build artifacts
	// ready to push remotely.
	deployLocalBuild deployStage = iota
	// deployPush pushes local build artifacts to the remote service, but doesn't
	// enable them.
	deployPush deployStage = iota
	// deployCommit commits the pushed artifacts to the remote service, activating
	// them.
	deployCommit deployStage = iota
	)

	var deployStageFlagEnum = flagenum.Enum{
	"stage": deployStaging,
	"localbuild": deployLocalBuild,
	"push": deployPush,
	"commit": deployCommit,
	}

	func (s *deployStage) Set(v string) error { return deployStageFlagEnum.FlagSet(s, v) }
	func (s deployStage) String() string { return deployStageFlagEnum.FlagString(s) }

	type deployParams struct {
	// stage is the highest deployment stage to deploy through.
	stage deployStage

	// ignoreCurrentVersion, if true, instructs depoyment logic to proceed with
	// deployment even if the currently-deployed version matches the current
	// source version.
	ignoreCurrentVersion bool
	// commitTainted, if true, instructs the deploy tool to perform commit
	// operations on tainted Components.
	commitTainted bool
	// forceVersion forces the use of the supplied version instead of the one
	// generated by the source.
	forceVersion cloudProjectVersion
	}

	var cmdDeploy = subcommands.Command{
	UsageLine: "deploy [DEPLOYMENT.[.COMPONENT]]...",
	ShortDesc: "Deploys some or all of a deployment.",
	LongDesc: "Deploys a Deployment. If specific projects are named, only those projects will be deployed.",
	CommandRun: func() subcommands.CommandRun {
	var cmd cmdDeployRun
	cmd.dp.params.stage = deployCommit

	fs := cmd.GetFlags()
	fs.BoolVar(&cmd.checkout, "checkout", false,
	"Refresh the checkout prior to deployment.")
	fs.Var(&cmd.dp.params.stage, "stage",
	"Run the deployment sequence up through this stage ["+deployStageFlagEnum.Choices()+"].")
	fs.BoolVar(&cmd.dp.params.commitTainted, "commit-tainted", false,
	"Install new deployments even if their source is tained.")
	fs.BoolVar(&cmd.dp.params.ignoreCurrentVersion, "ignore-current-version", false,
	"Push/install new deployments even if the currently-deployed version would not change or could not be parsed.")
	fs.StringVar(&cmd.manualVersion, "version", "",
	"Manually specify the version to use for deployment. Implies tainted.")
	return &cmd
	},
	}

	type cmdDeployRun struct {
	subcommands.CommandRunBase

	// checkout, if true, refreshes the checkout prior to deployment.
	checkout bool
	// manualVersion is the manually-specified cloud project version.
	manualVersion string

	dp deploymentPlan
	}

	func (cmd *cmdDeployRun) Run(app subcommands.Application, args []string, env subcommands.Env) int {
	a, c := app.(*application), cli.GetContext(app, cmd, env)

	// If a manual version was specified, use it.
	if cmd.manualVersion != "" {
	cmd.dp.params.forceVersion = makeStringCloudProjectVersion(cmd.manualVersion)
	}

	err := a.runWork(c, func(w *work) error {
	// Perform our planned checkout.
	if cmd.checkout {
	// Perform a full checkout.
	if err := checkout(w, &a.layout, false); err != nil {
	return errors.Annotate(err, "failed to checkout sources").Err()
	}
	}

	// Load our frozen checkout.
	if err := a.layout.loadFrozenLayout(w); err != nil {
	return errors.Annotate(err, "failed to load frozen checkout").Err()
	}
	return nil
	})
	if err != nil {
	logError(c, err, "Failed to perform checkout.")
	return 1
	}

	// Figure out what we're going to deploy.
	for _, arg := range args {
	matched := false
	err := a.layout.matchDeploymentComponent(arg, func(dep layoutDeployment, comp layoutDeploymentComponent) {
	matched = true

	if comp != nil {
	cmd.dp.addProjectComponent(comp)
	} else {
	cmd.dp.addDeployment(dep)
	}
	})
	if err != nil {
	logError(c, err, "Failed during matching.")
	return 1
	}
	if !matched {
	log.Fields{
	"target": arg,
	}.Errorf(c, "Deploy target did not match any configured deployments or components.")
	return 1
	}
	}

	err = a.runWork(c, func(w *work) error {
	if err := cmd.dp.initialize(w, &a.layout); err != nil {
	return errors.Annotate(err, "failed to initialize").Err()
	}

	return cmd.dp.deploy(w)
	})
	if err != nil {
	logError(c, err, "Failed to perform deployment.")
	return 1
	}
	return 0
	}

	type deploymentPlan struct {
	// comps maps DEPLOYMENT/PROJECT/COMPONENT to its specific deployable project
	// component.
	comps map[string]*layoutDeploymentComponent

	// deployments tracks which deployments we've specified.
	deployments map[title]struct{}

	// params are the deployment parameters to forward to deployment operations.
	params deployParams

	// reg is the registry of deployable Components.
	reg deploymentRegistry

	// layout is the deployment layout. It is initialized during "initialize".
	layout *deployLayout
	}

	func (dp deploymentPlan) addProjectComponent(comp layoutDeploymentComponent) {
	// Mark this specific component.
	if dp.comps == nil {
	dp.comps = make(map[string]*layoutDeploymentComponent)
	}
	dp.comps[comp.String()] = comp

	// Mark that we've used this deployment.
	if dp.deployments == nil {
	dp.deployments = make(map[title]struct{})
	}
	dp.deployments[comp.dep.title] = struct{}{}
	}

	func (dp deploymentPlan) addDeployment(dep layoutDeployment) {
	for _, comp := range dep.components {
	dp.addProjectComponent(comp)
	}
	}

	func (dp *deploymentPlan) deployedComponents() []string {
	comps := make([]string, 0, len(dp.comps))
	for path := range dp.comps {
	comps = append(comps, path)
	}
	sort.Strings(comps)
	return comps
	}

	func (dp deploymentPlan) initialize(w work, l *deployLayout) error {
	dp.layout = l

	// Load all deployable Components from checkout.
	for _, depName := range l.deploymentNames {
	dep := l.deployments[depName]

	if _, ok := dp.deployments[dep.title]; !ok {
	continue
	}

	// Load the application components. We will iterate through them in the
	// order in which they are defined in the Application's protobuf.
	for _, compName := range dep.componentNames {
	comp := dep.components[compName]

	// Only pass the registrar if this component is being deployed.
	var pReg componentRegistrar
	if _, ok := dp.comps[comp.String()]; ok {
	pReg = &dp.reg
	}
	if err := comp.loadSourceComponent(pReg); err != nil {
	return errors.Annotate(err, "failed to load component %q", comp).Err()
	}
	}
	}

	return nil
	}

	func (dp deploymentPlan) deploy(w work) error {
	// Create our working root and staging/build subdirectories.
	fs, err := dp.layout.workingFilesystem()
	if err != nil {
	return errors.Annotate(err, "failed to create working directory").Err()
	}

	stagingDir, err := fs.Base().EnsureDirectory("staging")
	if err != nil {
	return errors.Annotate(err, "failed to create staging directory").Err()
	}

	// Stage: Staging
	if err := dp.reg.stage(w, stagingDir, &dp.params); err != nil {
	return errors.Annotate(err, "failed to stage").Err()
	}
	if dp.params.stage <= deployStaging {
	return nil
	}

	// Stage: Local Build
	if err := dp.reg.localBuild(w); err != nil {
	return errors.Annotate(err, "failed to perform local build").Err()
	}
	if dp.params.stage <= deployLocalBuild {
	return nil
	}

	// Stage: Push
	if err := dp.reg.push(w); err != nil {
	return errors.Annotate(err, "failed to push components").Err()
	}
	if dp.params.stage <= deployPush {
	return nil
	}

	// Stage: Commit
	if err := dp.reg.commit(w); err != nil {
	return errors.Annotate(err, "failed to commit").Err()
	}
	return nil
	}

	// deploymentRegistry is the registry of all prepared deployments.
	//
	// While deployment is specified and staged at a component level, actual
	// deployment is allowed to happen at a practical level (e.g., a single
	// cloud project's AppEngine configuration, a single Google Container Engine
	// cluster, etc.).
	//
	// This will track individual components, as well as integrate them into larger
	// action plans.
	type deploymentRegistry struct {
	// components is a map of components that have been deployed.
	components map[string]*layoutDeploymentComponent
	// componentNames is the sorted list of component names in components.
	componentNames []string

	// gaeProjects maps Google AppEngine deployment state to specific GAE
	// projects.
	gaeProjects map[string]*gaeDeployment
	// gaeProjectNames is the sorted list of registered GAE project names.
	gaeProjectNames []string

	// gkeProjects is the set of Google Container Engine deployments for a given
	// cloud project.
	gkeProjects map[string]*containerEngineDeployment
	// gkeProjectNames is the sorted list of registered GKE project names.
	gkeProjectNames []string
	}

	func (reg deploymentRegistry) addComponent(comp layoutDeploymentComponent) {
	name := comp.comp.Name
	if _, has := reg.components[name]; has {
	return
	}

	if reg.components == nil {
	reg.components = make(map[string]*layoutDeploymentComponent)
	}
	reg.components[name] = comp
	reg.componentNames = append(reg.componentNames, name)
	}

	func (reg deploymentRegistry) addGAEModule(module layoutDeploymentGAEModule) {
	// Get/create the AppEngine project for this module.
	cloudProjectName := module.comp.dep.cloudProject.Name
	gaeD := reg.gaeProjects[cloudProjectName]
	if gaeD == nil {
	gaeD = makeGAEDeployment(module.comp.dep.cloudProject)

	if reg.gaeProjects == nil {
	reg.gaeProjects = make(map[string]*gaeDeployment)
	}
	reg.gaeProjects[cloudProjectName] = gaeD
	reg.gaeProjectNames = append(reg.gaeProjectNames, cloudProjectName)
	}

	gaeD.addModule(module)
	reg.addComponent(module.comp)
	}

	func (reg deploymentRegistry) addGKEPod(pb layoutDeploymentGKEPodBinding) {
	// Get/create the AppEngine project for this module.
	cloudProjectName := pb.cluster.cloudProject.Name
	gkeD := reg.gkeProjects[cloudProjectName]
	if gkeD == nil {
	gkeD = &containerEngineDeployment{
	project: pb.cluster.cloudProject,
	clusters: make(map[string]*containerEngineDeploymentCluster),
	}

	if reg.gkeProjects == nil {
	reg.gkeProjects = make(map[string]*containerEngineDeployment)
	}
	reg.gkeProjects[cloudProjectName] = gkeD
	reg.gkeProjectNames = append(reg.gkeProjectNames, cloudProjectName)
	}

	// Register this pod in its cluster.
	gkeD.addCluster(pb.cluster).attachPod(pb)
	}

	// appEngineModulesOnly clears all deployment parameters that are not AppEngine
	// modules.
	func (reg *deploymentRegistry) appEngineModulesOnly() {
	reg.gkeProjects, reg.gkeProjectNames = nil, nil
	}

	// stage performs staging, preparing deployment structures on the local
	// filesystem.
	func (reg deploymentRegistry) stage(w work, stageDir managedfs.Dir, params deployParams) error {
	// Sort our name lists.
	sort.Strings(reg.componentNames)
	sort.Strings(reg.gaeProjectNames)
	sort.Strings(reg.gkeProjectNames)

	// All components can be independently staged in parallel.
	//
	// We will still enumerate over them in sorted order for determinism.
	return w.RunMulti(func(workC chan<- func() error) {
	// AppEngine Projects
	for _, name := range reg.gaeProjectNames {
	proj := reg.gaeProjects[name]
	workC <- func() error {
	depDir, err := stageDir.EnsureDirectory(string(proj.project.dep.title), "appengine")
	if err != nil {
	return errors.Annotate(err, "failed to create deployment directory for %q", proj.project.dep.title).Err()
	}
	return proj.stage(w, depDir, params)
	}
	}

	// Container Engine Projects
	for _, name := range reg.gkeProjectNames {
	proj := reg.gkeProjects[name]
	workC <- func() error {
	depDir, err := stageDir.EnsureDirectory(string(proj.project.dep.title), "container_engine")
	if err != nil {
	return errors.Annotate(err, "failed to create deployment directory for %q", proj.project.dep.title).Err()
	}
	return proj.stage(w, depDir, params)
	}
	}
	})
	}

	// build performs the build stage on staged components.
	func (reg deploymentRegistry) localBuild(w work) error {
	// Enumerate over them in sorted order for determinism.
	return w.RunMulti(func(workC chan<- func() error) {
	// AppEngine Projects
	for _, name := range reg.gaeProjectNames {
	proj := reg.gaeProjects[name]
	workC <- func() error {
	return proj.localBuild(w)
	}
	}

	// Container Engine Projects
	for _, name := range reg.gkeProjectNames {
	proj := reg.gkeProjects[name]
	workC <- func() error {
	return proj.localBuild(w)
	}
	}
	})
	}

	// push builds and pushes the deployments to remote services, but does not
	// commit to them.
	func (reg deploymentRegistry) push(w work) error {
	// All components can be independently pushed in parallel.
	//
	// We will still enumerate over them in sorted order for determinism.
	return w.RunMulti(func(workC chan<- func() error) {
	// AppEngine Projects
	for _, name := range reg.gaeProjectNames {
	proj := reg.gaeProjects[name]
	workC <- func() error {
	return proj.push(w)
	}
	}

	// Container Engine Projects
	for _, name := range reg.gkeProjectNames {
	proj := reg.gkeProjects[name]
	workC <- func() error {
	return proj.push(w)
	}
	}
	})
	}

	// commit commits the deployment to the remote services.
	func (reg deploymentRegistry) commit(w work) error {
	// All components can be independently committed in parallel.
	//
	// We will still enumerate over them in sorted order for determinism.
	return w.RunMulti(func(workC chan<- func() error) {
	// AppEngine Projects
	for _, name := range reg.gaeProjectNames {
	proj := reg.gaeProjects[name]
	workC <- func() error {
	return proj.commit(w)
	}
	}

	// Container Engine Projects
	for _, name := range reg.gkeProjectNames {
	proj := reg.gkeProjects[name]
	workC <- func() error {
	return proj.commit(w)
	}
	}
	})
	}