in cmd/hub/lifecycle/deploy.go [42:592]
func Execute(request *Request, pipe io.WriteCloser) {
isDeploy := strings.HasPrefix(request.Verb, "deploy")
isUndeploy := strings.HasPrefix(request.Verb, "undeploy")
isSomeComponents := len(request.Components) > 0 || request.OffsetComponent != ""
stackManifest, componentsManifests, chosenManifestFilename, err := manifest.ParseManifest(request.ManifestFilenames)
if err != nil {
log.Fatalf("Unable to %s: %s", request.Verb, err)
}
if pipe != nil {
metricTags := fmt.Sprintf("stack:%s", stackManifest.Meta.Name)
pipe.Write([]byte(metricTags))
pipe.Close()
}
environment, err := util.ParseKvList(request.EnvironmentOverrides)
if err != nil {
log.Fatalf("Unable to parse environment settings `%s`: %v", request.EnvironmentOverrides, err)
}
order, err := manifest.GenerateLifecycleOrder(stackManifest)
if err != nil {
log.Fatal(err)
}
stackManifest.Lifecycle.Order = order
if config.Verbose {
printStartBlurb(request, chosenManifestFilename, stackManifest)
}
stackBaseDir := util.Basedir(request.ManifestFilenames)
componentsBaseDir := request.ComponentsBaseDir
if componentsBaseDir == "" {
componentsBaseDir = stackBaseDir
}
components := stackManifest.Components
checkComponentsManifests(components, componentsManifests)
checkLifecycleOrder(components, stackManifest.Lifecycle)
checkLifecycleRequires(components, stackManifest.Lifecycle.Requires)
checkComponentsDepends(components, stackManifest.Lifecycle.Order)
manifest.CheckComponentsExist(components, append(request.Components, request.OffsetComponent, request.LimitComponent)...)
optionalRequires := parseRequiresTunning(stackManifest.Lifecycle.Requires)
requiresOfOptionalComponents := calculateRequiresOfOptionalComponents(componentsManifests, &stackManifest.Lifecycle, stackManifest.Requires)
stackRequires := maybeOmitCloudRequires(stackManifest.Requires, request.EnabledClouds)
provides := checkStackRequires(stackRequires, optionalRequires, requiresOfOptionalComponents)
mergePlatformProvides(provides, stackManifest.Platform.Provides)
if config.Debug && len(provides) > 0 {
log.Print("Requirements provided by:")
util.PrintDeps(provides)
}
osEnv, err := initOsEnv(request.OsEnvironmentMode)
if err != nil {
log.Fatalf("Unable to parse OS environment setup: %v", err)
}
defer util.Done()
var stateManifest *state.StateManifest
var operationsHistory []state.LifecycleOperation
stateUpdater := func(interface{}) {}
var operationLogId string
if len(request.StateFilenames) > 0 {
stateFiles, errs := storage.Check(request.StateFilenames, "state")
if len(errs) > 0 {
util.MaybeFatalf("Unable to check state files: %s", util.Errors2(errs...))
}
storage.EnsureNoLockFiles(stateFiles)
parsed, err := state.ParseState(stateFiles)
if isUndeploy || isSomeComponents {
if err != nil {
if err != os.ErrNotExist {
log.Fatalf("Failed to read %v state files: %v", request.StateFilenames, err)
}
if isSomeComponents {
comps := request.OffsetComponent
if comps == "" {
comps = strings.Join(request.Components, ", ")
}
util.MaybeFatalf("Component `%s` is specified but failed to read %v state file(s): %v",
comps, request.StateFilenames, err)
}
} else {
stateManifest = parsed
}
} else { // full deploy copy state oplog if possible
if err == nil && parsed != nil {
if config.Debug {
log.Print("Preserving operations history loaded from existing state")
}
operationsHistory = parsed.Operations
if parsed.Meta.Name != "" && parsed.Meta.Name != stackManifest.Meta.Name {
util.Warn("State meta.name = `%s` does not match elaborate meta.name = `%s`",
parsed.Meta.Name, stackManifest.Meta.Name)
}
stateManifest = parsed
}
}
var syncer func(*state.StateManifest)
// TODO sync status if no state manifest on undeploy
if request.SyncStackInstance && request.StackInstance != "" {
syncer = hubSyncer(request)
}
stateUpdater = state.InitWriter(stateFiles, syncer)
u, err := uuid.NewRandom()
if err != nil {
log.Fatalf("Unable to generate operation Id random v4 UUID: %v", err)
}
operationLogId = u.String()
}
deploymentIdParameterName := "hub.deploymentId"
deploymentId := ""
if stateManifest != nil {
for _, p := range stateManifest.StackParameters {
if p.Name == deploymentIdParameterName {
deploymentId = util.String(p.Value)
break
}
}
}
if deploymentId == "" {
u, err := uuid.NewRandom()
if err != nil {
log.Fatalf("Unable to generate `hub.deploymentId` random v4 UUID: %v", err)
}
deploymentId = u.String()
}
stackNameParameterName := "hub.stackName"
stackName := ""
if stateManifest != nil {
for _, p := range stateManifest.StackParameters {
if p.Name == stackNameParameterName {
stackName = util.String(p.Value)
break
}
}
}
if stackName == "" {
stackName = os.Getenv(HubEnvVarHubStackName)
if stackName == "" {
suffix := rand.Intn(1000) + 1
name := petname.Generate(2, "-")
stackName = fmt.Sprintf("%s-%d", name, suffix)
}
}
extraExpansionValues := []manifest.Parameter{
{Name: deploymentIdParameterName, Value: deploymentId},
{Name: stackNameParameterName, Value: stackName},
}
// TODO state file has user-level parameters for undeploy operation
// should we just go with the state values if we cannot lock all parameters properly?
stackParameters, errs := parameters.LockParameters(
manifest.FlattenParameters(stackManifest.Parameters, chosenManifestFilename),
extraExpansionValues,
func(parameter manifest.Parameter) (interface{}, error) {
return AskParameter(parameter, environment,
request.Environment, request.StackInstance, request.Application,
isDeploy)
})
if len(errs) > 0 {
log.Fatalf("Failed to lock stack parameters:\n\t%s", util.Errors("\n\t", errs...))
}
allOutputs := make(parameters.CapturedOutputs)
if stateManifest != nil {
checkStateMatch(stateManifest, stackManifest, stackParameters)
state.MergeParsedStateParametersAndProvides(stateManifest, stackParameters, provides)
if isUndeploy && !isSomeComponents {
state.MergeParsedStateOutputs(stateManifest,
"", []string{}, stackManifest.Lifecycle.Order, false,
allOutputs)
}
}
if stateManifest == nil && isDeploy {
stateManifest = &state.StateManifest{
Meta: state.Metadata{
Kind: stackManifest.Kind,
Name: stackManifest.Meta.Name,
},
Operations: operationsHistory,
}
}
addLockedParameter(stackParameters, deploymentIdParameterName, "DEPLOYMENT_ID", deploymentId)
addLockedParameter(stackParameters, stackNameParameterName, "STACK_NAME", stackName)
stackParametersNoLinks := parameters.ParametersWithoutLinks(stackParameters)
order = stackManifest.Lifecycle.Order
if stateManifest != nil {
stateManifest.Lifecycle.Order = order
}
offsetGuessed := false
if isUndeploy {
order = util.Reverse(order)
// on undeploy, guess which component failed to deploy and start undeploy from it
if request.GuessComponent && stateManifest != nil && !isSomeComponents {
for i, component := range order {
if _, exist := stateManifest.Components[component]; exist {
if i == 0 {
break
}
if config.Verbose {
log.Printf("State file has a state for `%[1]s` - setting `--offset %[1]s`", component)
}
request.OffsetComponent = component
offsetGuessed = true
break
}
}
}
}
offsetComponentIndex := util.Index(order, request.OffsetComponent)
limitComponentIndex := util.Index(order, request.LimitComponent)
if offsetComponentIndex >= 0 && limitComponentIndex >= 0 &&
limitComponentIndex < offsetComponentIndex && !offsetGuessed {
log.Fatalf("Specified --limit %s (%d) is before specified --offset %s (%d) in component order",
request.LimitComponent, limitComponentIndex, request.OffsetComponent, offsetComponentIndex)
}
skipComponent := func(i int, name string) bool {
return (len(request.Components) > 0 && !util.Contains(request.Components, name)) ||
(offsetComponentIndex >= 0 && i < offsetComponentIndex) ||
(limitComponentIndex >= 0 && i > limitComponentIndex)
}
checkComponentsSourcesExist(order, components, stackBaseDir, componentsBaseDir, skipComponent)
checkLifecycleVerbs(order, components, componentsManifests, stackManifest.Lifecycle.Verbs, stackBaseDir, componentsBaseDir, skipComponent)
failedComponents := make([]string, 0)
// TODO handle ^C interrupt to update op log and stack status
// or expiry by time and set to `interrupted`
if stateManifest != nil {
stateManifest = state.UpdateOperation(stateManifest, operationLogId, request.Verb, "in-progress",
map[string]interface{}{"args": os.Args})
}
ctx := watchInterrupt()
NEXT_COMPONENT:
for componentIndex, componentName := range order {
if skipComponent(componentIndex, componentName) {
if config.Debug {
log.Printf("Skip %s", componentName)
}
continue
}
if config.Verbose {
log.Printf(util.HighlightColor("%s ***%s*** (%d/%d)"), maybeTestVerb(request.Verb, request.DryRun),
componentName, componentIndex+1, len(components))
}
component := manifest.ComponentRefByName(components, componentName)
componentManifest := manifest.ComponentManifestByRef(componentsManifests, component)
if stateManifest != nil && (componentIndex == offsetComponentIndex || len(request.Components) > 0) {
if len(request.Components) > 0 {
allOutputs = make(parameters.CapturedOutputs)
}
state.MergeParsedStateOutputs(stateManifest,
componentName, component.Depends, stackManifest.Lifecycle.Order, isDeploy,
allOutputs)
}
var updateStateComponentFailed func(string, bool)
if stateManifest != nil {
stateManifest = state.UpdateComponentStartTimestamp(stateManifest, componentName)
updateStateComponentFailed = func(msg string, final bool) {
stateManifest = state.UpdateComponentStatus(stateManifest, componentName, &componentManifest.Meta, "error", msg)
stateManifest = state.UpdatePhase(stateManifest, operationLogId, componentName, "error")
// Erasing provides of a failed component on redeploy has undesirable effect on undeploy, for example:
// Kubernetes Terraform failed safely - failed to download plugin, or failed to add minor resource - leaving
// Kubernetes fully operation, yet the `kubernetes` capability is removed from stack provides. Such stack
// will fail to undeploy due to components being unable to satisfy the [kubernetes] requirement, ie.
// requiring deploy to undeploy, which is dumb.
// eraseProvides(provides, componentName)
stateManifest.Provides = noEnvironmentProvides(provides)
if !config.Force && !optionalComponent(&stackManifest.Lifecycle, componentName) {
stateManifest = state.UpdateStackStatus(stateManifest, "incomplete", msg)
}
if final {
stateManifest = state.UpdateOperation(stateManifest, operationLogId, request.Verb, "error", nil)
}
stateUpdater(stateManifest)
}
}
if isDeploy && len(component.Depends) > 0 {
failed := make([]string, 0, len(component.Depends))
for _, dependency := range component.Depends {
if util.Contains(failedComponents, dependency) {
failed = append(failed, dependency)
}
}
if len(failed) > 0 {
maybeFatalIfMandatory(&stackManifest.Lifecycle, componentName,
fmt.Sprintf("Component `%s` failed to %s: depends on failed optional component `%s`",
componentName, request.Verb, strings.Join(failed, ", ")),
updateStateComponentFailed)
failedComponents = append(failedComponents, componentName)
continue NEXT_COMPONENT
}
}
expandedComponentParameters, expansionErrs := parameters.ExpandParameters(componentName, componentManifest.Meta.Kind, component.Depends,
stackParameters, allOutputs,
manifest.FlattenParameters(componentManifest.Parameters, componentManifest.Meta.Name))
expandedComponentParameters = addHubProvides(expandedComponentParameters, provides)
allParameters := parameters.MergeParameters(stackParametersNoLinks, expandedComponentParameters)
optionalParametersFalse := calculateOptionalFalseParameters(componentName, allParameters, optionalRequires)
if len(optionalParametersFalse) > 0 {
log.Printf("Surprisingly, let skip `%s` due to optional parameter %v evaluated to false", componentName, optionalParametersFalse)
if stateManifest != nil {
stateManifest = state.EraseComponentEmptyState(stateManifest, componentName)
}
continue NEXT_COMPONENT
}
if len(expansionErrs) > 0 {
log.Printf("Component `%s` failed to %s", componentName, request.Verb)
maybeFatalIfMandatory(&stackManifest.Lifecycle, componentName,
fmt.Sprintf("Component `%s` parameters expansion failed:\n\t%s",
componentName, util.Errors("\n\t", expansionErrs...)),
updateStateComponentFailed)
failedComponents = append(failedComponents, componentName)
continue NEXT_COMPONENT
}
componentParameters := parameters.MergeParameters(make(parameters.LockedParameters), expandedComponentParameters)
if optionalNotProvided, err := prepareComponentRequires(provides, componentManifest, allParameters, allOutputs, optionalRequires, request.EnabledClouds); len(optionalNotProvided) > 0 || err != nil {
if err != nil {
if request.Verb == "undeploy" {
// proceed without --force set to handle required component (depends on) being already undeployed via --component
util.Warn("%v", err)
} else {
maybeFatalIfMandatory(&stackManifest.Lifecycle, componentName, fmt.Sprintf("%v", err), updateStateComponentFailed)
continue NEXT_COMPONENT
}
}
if len(optionalNotProvided) > 0 {
log.Printf("Skip %s due to unsatisfied optional requirements %v", componentName, optionalNotProvided)
// there will be a gap in state file but `deploy -c` will be able to find some state from
// a preceding component
continue NEXT_COMPONENT
}
}
if stateManifest != nil {
if isDeploy {
stateManifest = state.UpdateState(stateManifest, componentName,
stackParameters, expandedComponentParameters,
nil, allOutputs, stackManifest.Outputs,
noEnvironmentProvides(provides),
false)
}
status := fmt.Sprintf("%sing", request.Verb)
stateManifest = state.UpdateComponentStatus(stateManifest, componentName, &componentManifest.Meta, status, "")
stateManifest = state.UpdateStackStatus(stateManifest, status, "")
stateManifest = state.UpdatePhase(stateManifest, operationLogId, componentName, "in-progress")
stateUpdater(stateManifest)
stateUpdater("sync")
}
randomSize := 128
if opts := componentManifest.Lifecycle.Options; opts != nil {
if rnd := opts.Random; rnd != nil && rnd.Bytes > 0 {
randomSize = rnd.Bytes
}
}
randomStr, random, err := util.Random(randomSize)
if err != nil {
util.Warn("Unable to set %s: %v", HubEnvVarNameRandom, err)
}
componentDir := manifest.ComponentSourceDirFromRef(component, stackBaseDir, componentsBaseDir)
verb := maybeTestVerb(request.Verb, request.DryRun)
preHookVerb := fmt.Sprintf("pre-%s", verb)
stdout, stderr, err := fireHooks(preHookVerb, stackBaseDir, component, componentParameters, osEnv)
if err != nil {
if stateManifest != nil && request.WriteOplogToStateOnError {
stateManifest = state.AppendOperationLog(stateManifest, operationLogId,
fmt.Sprintf("%v%s", err, formatStdoutStderr(stdout, stderr)))
}
util.MaybeFatalf2(updateStateComponentFailed, "One of %s hooks failed. See logs", preHookVerb)
}
stdout, stderr, err = delegate(verb,
component, componentManifest, componentParameters,
componentDir, osEnv, randomStr, stackBaseDir)
var rawOutputs parameters.RawOutputs
if err != nil {
if stateManifest != nil && request.WriteOplogToStateOnError {
stateManifest = state.AppendOperationLog(stateManifest, operationLogId,
fmt.Sprintf("%v%s", err, formatStdoutStderr(stdout, stderr)))
}
maybeFatalIfMandatory(&stackManifest.Lifecycle, componentName,
fmt.Sprintf("Component `%s` failed to %s: %v", componentName, request.Verb, err),
updateStateComponentFailed)
failedComponents = append(failedComponents, componentName)
} else if isDeploy {
rawOutputsCaptured, componentOutputs, dynamicProvides, errs := captureOutputs(componentName, componentDir, componentManifest, componentParameters,
stdout, random)
rawOutputs = rawOutputsCaptured
if len(errs) > 0 {
log.Printf("Component `%s` failed to %s", componentName, request.Verb)
maybeFatalIfMandatory(&stackManifest.Lifecycle, componentName,
fmt.Sprintf("Component `%s` outputs capture failed:\n\t%s",
componentName, util.Errors("\n\t", errs...)),
updateStateComponentFailed)
failedComponents = append(failedComponents, componentName)
}
if len(componentOutputs) > 0 &&
(config.Debug || (config.Verbose && len(request.Components) == 1)) {
log.Print("Component outputs:")
parameters.PrintCapturedOutputs(componentOutputs)
}
parameters.MergeOutputs(allOutputs, componentOutputs)
if request.GitOutputs {
if config.Debug || (config.Verbose && request.GitOutputsStatus) {
log.Print("Checking Git status")
}
git := gitOutputs(componentName, componentDir, request.GitOutputsStatus)
if config.Debug && len(git) > 0 {
log.Print("Implicit Git outputs added:")
parameters.PrintCapturedOutputs(git)
}
parameters.MergeOutputs(allOutputs, git)
}
componentComplexOutputs := captureProvides(component, stackBaseDir, componentsBaseDir,
componentManifest.Provides, componentOutputs)
if len(componentComplexOutputs) > 0 &&
(config.Debug || (config.Verbose && len(request.Components) == 1)) {
log.Print("Component additional outputs captured:")
parameters.PrintCapturedOutputs(componentComplexOutputs)
}
parameters.MergeOutputs(allOutputs, componentComplexOutputs)
mergeProvides(provides, componentName, append(dynamicProvides, componentManifest.Provides...), componentOutputs)
} else if isUndeploy {
eraseProvides(provides, componentName)
if stateManifest != nil {
stateManifest.Provides = noEnvironmentProvides(provides)
}
}
postHookVerb := fmt.Sprintf("post-%s", verb)
stdout, stderr, err = fireHooks(postHookVerb, stackBaseDir, component, componentParameters, osEnv)
if err != nil {
if stateManifest != nil && request.WriteOplogToStateOnError {
stateManifest = state.AppendOperationLog(stateManifest, operationLogId,
fmt.Sprintf("%v%s", err, formatStdoutStderr(stdout, stderr)))
}
util.MaybeFatalf2(updateStateComponentFailed, "One of %s hooks failed. See logs", postHookVerb)
}
if ctx.Err() != nil {
break
}
if stateManifest != nil && isDeploy {
final := componentIndex == len(order)-1 || (len(request.Components) > 0 && request.LoadFinalState)
stateManifest = state.UpdateState(stateManifest, componentName,
stackParameters, expandedComponentParameters,
rawOutputs, allOutputs, stackManifest.Outputs,
noEnvironmentProvides(provides), final)
}
if err == nil && isDeploy {
err = waitForReadyConditions(ctx, componentManifest.Lifecycle.ReadyConditions, componentParameters, allOutputs, component.Depends)
if err != nil {
log.Printf("Component `%s` failed to %s", componentName, request.Verb)
maybeFatalIfMandatory(&stackManifest.Lifecycle, componentName,
fmt.Sprintf("Component `%s` ready condition failed: %v", componentName, err),
updateStateComponentFailed)
failedComponents = append(failedComponents, componentName)
}
}
if err == nil && config.Verbose {
log.Printf("Component `%s` completed %s", componentName, request.Verb)
}
if stateManifest != nil {
if !util.Contains(failedComponents, componentName) {
stateManifest = state.UpdateComponentStatus(stateManifest, componentName, &componentManifest.Meta,
fmt.Sprintf("%sed", request.Verb), "")
stateManifest = state.UpdatePhase(stateManifest, operationLogId, componentName, "success")
stateUpdater(stateManifest)
}
}
// end of component cycle
}
stackReadyConditionFailed := false
if isDeploy {
err := waitForReadyConditions(ctx, stackManifest.Lifecycle.ReadyConditions, stackParameters, allOutputs, nil)
if err != nil {
message := fmt.Sprintf("Stack ready condition failed: %v", err)
if stateManifest != nil {
stateManifest = state.UpdateStackStatus(stateManifest, "incomplete", message)
stateManifest = state.UpdateOperation(stateManifest, operationLogId, request.Verb, "error", nil)
stateUpdater(stateManifest)
}
util.MaybeFatalf("%s", message)
stackReadyConditionFailed = true
}
}
if stateManifest != nil {
if !stackReadyConditionFailed {
status, message := calculateStackStatus(stackManifest, stateManifest, request.Verb)
stateManifest = state.UpdateStackStatus(stateManifest, status, message)
stateManifest = state.UpdateOperation(stateManifest, operationLogId, request.Verb, "success", nil)
stateUpdater(stateManifest)
}
stateUpdater("sync")
}
var stackOutputs []parameters.ExpandedOutput
if stateManifest != nil {
stackOutputs = stateManifest.StackOutputs
} else {
stackOutputs = parameters.ExpandRequestedOutputs(stackParameters, allOutputs, stackManifest.Outputs, false)
}
if config.Verbose {
if isDeploy {
provides2 := noEnvironmentProvides(provides)
if len(provides2) > 0 {
log.Printf("%s provides:", cases.Title(language.Und).String(stackManifest.Kind))
util.PrintDeps(provides2)
}
printStackOutputs(stackOutputs)
}
}
if config.Verbose {
printEndBlurb(request, stackManifest)
}
}