diff --git a/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt b/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt index 7d830c172b1..b80297a767d 100644 --- a/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt +++ b/acceptance/bundle/deploy/wal/failed-plan-no-wal/output.txt @@ -16,8 +16,6 @@ HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. -Error: planning failed - Exit code: 1 @@ -33,8 +31,6 @@ HTTP Status: 403 Forbidden API error_code: INJECTED API message: Fault injected by test. -Error: planning failed - Exit code: 1 diff --git a/acceptance/bundle/lifecycle/prevent-destroy/out.direct.txt b/acceptance/bundle/lifecycle/prevent-destroy/out.direct.txt index a100c8774f7..46c8c33394d 100644 --- a/acceptance/bundle/lifecycle/prevent-destroy/out.direct.txt +++ b/acceptance/bundle/lifecycle/prevent-destroy/out.direct.txt @@ -1,7 +1,8 @@ >>> musterr [CLI] bundle destroy --auto-approve Error: resources.pipelines.my_pipelines has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for resources.pipelines.my_pipelines -resources.schemas.my_schema has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for resources.schemas.my_schema + +Error: resources.schemas.my_schema has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for resources.schemas.my_schema >>> errcode [CLI] bundle plan @@ -23,7 +24,8 @@ Plan: 2 to add, 0 to change, 2 to delete, 0 unchanged >>> musterr [CLI] bundle deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/prevent-destroy/default/files... Error: resources.pipelines.my_pipelines has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for resources.pipelines.my_pipelines -resources.schemas.my_schema has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for resources.schemas.my_schema + +Error: resources.schemas.my_schema has lifecycle.prevent_destroy set, but the plan calls for this resource to be recreated or destroyed. To avoid this error, disable lifecycle.prevent_destroy for resources.schemas.my_schema >>> errcode [CLI] bundle plan diff --git a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.plan.direct.txt b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.plan.direct.txt index e4eebfb055f..80230a2f0fd 100644 --- a/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.plan.direct.txt +++ b/acceptance/bundle/resource_deps/bad_ref_string_to_int/out.plan.direct.txt @@ -2,5 +2,3 @@ >>> musterr [CLI] bundle plan Error: cannot plan resources.jobs.foo: cannot update tasks[0].run_job_task.job_id with value of "${resources.jobs.bar.name}": cannot set (*jobs.JobSettings).tasks[0].run_job_task.job_id to string ("job bar"): cannot parse "job bar" as int64: strconv.ParseInt: parsing "job bar": invalid syntax -Error: planning failed - diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt index 841f33c1d2f..109e5ebb857 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.deploy.direct.txt @@ -1,5 +1,3 @@ Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo -Error: planning failed - diff --git a/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt b/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt index 6a2b5d1157c..c93979de60f 100644 --- a/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt +++ b/acceptance/bundle/resource_deps/non_existent_field/out.plan.direct.txt @@ -1,4 +1,2 @@ Error: cannot plan resources.volumes.foo: cannot resolve "${resources.volumes.bar.non_existent}": schema mismatch: non_existent: field "non_existent" not found in catalog.CreateVolumeRequestContent; non_existent: field "non_existent" not found in catalog.VolumeInfo -Error: planning failed - diff --git a/acceptance/bundle/resources/catalogs/auto-approve/output.txt b/acceptance/bundle/resources/catalogs/auto-approve/output.txt index e0576f1b9e5..4341e4cf264 100644 --- a/acceptance/bundle/resources/catalogs/auto-approve/output.txt +++ b/acceptance/bundle/resources/catalogs/auto-approve/output.txt @@ -39,8 +39,6 @@ test-file-[UNIQUE_NAME].txt -> dbfs:/Volumes/test-catalog-[UNIQUE_NAME]/test-sch Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/[UNIQUE_NAME]/files... Error: cannot plan resources.schemas.foo: cannot resolve "${resources.catalogs.bar.name}": internal error: resources.catalogs.bar: action is "delete" missing new_state -Error: planning failed - === Test cleanup >>> [CLI] bundle destroy --auto-approve diff --git a/acceptance/bundle/resources/permissions/pipelines/504/plan/output.txt b/acceptance/bundle/resources/permissions/pipelines/504/plan/output.txt index 6588ab497c6..e93f0d9d5c8 100644 --- a/acceptance/bundle/resources/permissions/pipelines/504/plan/output.txt +++ b/acceptance/bundle/resources/permissions/pipelines/504/plan/output.txt @@ -26,6 +26,4 @@ HTTP Status: 504 Gateway Timeout API error_code: INJECTED API message: Fault injected by test. -Error: planning failed - 3 {"method": "GET", "path": "/api/2.0/permissions/pipelines/[UUID]"} diff --git a/bundle/apps/validate.go b/bundle/apps/validate.go index 45a5b49c672..1df8b84a3df 100644 --- a/bundle/apps/validate.go +++ b/bundle/apps/validate.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/service/apps" ) @@ -17,7 +18,7 @@ var resourceReferencePattern = regexp.MustCompile(`\$\{resources\.(\w+)\.([^.]+) type validate struct{} -func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) error { var diags diag.Diagnostics usedSourceCodePaths := make(map[string]string) @@ -52,10 +53,10 @@ func (v *validate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } usedSourceCodePaths[app.SourceCodePath] = key - diags = diags.Extend(warnForAppResourcePermissions(b, key, app)) + warnForAppResourcePermissions(ctx, b, key, app) } - return diags + return logdiag.Flush(ctx, diags) } // appResourceRef extracts resource references from an app resource entry. @@ -130,9 +131,7 @@ func hasAppSPInPermissions(b *bundle.Bundle, resourcePath, appKey string) bool { // Without the SP in the permission list, the second deploy will overwrite the // app-granted permission on the resource. // See https://github.com/databricks/cli/issues/4309 -func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resources.App) diag.Diagnostics { - var diags diag.Diagnostics - +func warnForAppResourcePermissions(ctx context.Context, b *bundle.Bundle, appKey string, app *resources.App) { for _, ar := range app.Resources { ref, ok := appResourceRef(ar) if !ok { @@ -155,7 +154,7 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc } appPath := "resources.apps." + appKey - diags = append(diags, diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("app %q references %s %q which has permissions set. To prevent permission override after deploying the app, please add the app service principal to the %s permissions", appKey, refType, resourceKey, refType), Detail: fmt.Sprintf( @@ -176,8 +175,6 @@ func warnForAppResourcePermissions(b *bundle.Bundle, appKey string, app *resourc Locations: b.Config.GetLocations(appPath), }) } - - return diags } func (v *validate) Name() string { diff --git a/bundle/artifacts/build.go b/bundle/artifacts/build.go index 38b42dd1ab6..b9517b0c8d5 100644 --- a/bundle/artifacts/build.go +++ b/bundle/artifacts/build.go @@ -29,7 +29,7 @@ func (m *build) Name() string { return "artifacts.Build" } -func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *build) Apply(ctx context.Context, b *bundle.Bundle) error { cacheDir, err := b.LocalStateDir(ctx) if err != nil { logdiag.LogDiag(ctx, diag.Diagnostic{ @@ -44,23 +44,19 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if a.BuildCommand != "" { err := doBuild(ctx, artifactName, a) if err != nil { - logdiag.LogError(ctx, err) - break + return err } // We need to expand glob reference after build mutator is applied because // if we do it before, any files that are generated by build command will // not be included into artifact.Files and thus will not be uploaded. // We only do it if BuildCommand was specified because otherwise it should have been done already by artifacts.Prepare() - bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName}) + if err := bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName}); err != nil { + return err + } // After bundle.ApplyContext is called, all of b.Config is recreated and all pointers are invalidated (!) a = b.Config.Artifacts[artifactName] - - if logdiag.HasError(ctx) { - break - } - } if a.Type == "whl" && a.DynamicVersion && cacheDir != "" { @@ -71,9 +67,6 @@ func (m *build) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if patchedWheel != "" { a.Files[ind].Patched = patchedWheel } - if logdiag.HasError(ctx) { - break - } } } } diff --git a/bundle/artifacts/expand_globs.go b/bundle/artifacts/expand_globs.go index af358d3dda9..ee91ea7a174 100644 --- a/bundle/artifacts/expand_globs.go +++ b/bundle/artifacts/expand_globs.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/patchwheel" ) @@ -39,7 +40,7 @@ func (e expandGlobs) Name() string { return "expandGlobs" } -func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) error { // Base path for this mutator. // This path is set with the list of expanded globs when done. base := dyn.NewPath( @@ -117,8 +118,8 @@ func (e expandGlobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnosti return dyn.SetByPath(rootv, base, dyn.V(output)) }) if err != nil { - diags = diags.Extend(diag.FromErr(err)) + return err } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/artifacts/prepare.go b/bundle/artifacts/prepare.go index 9f8b2e6eaed..a55d001f6ce 100644 --- a/bundle/artifacts/prepare.go +++ b/bundle/artifacts/prepare.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/python" ) @@ -29,23 +28,22 @@ func (m *prepare) Name() string { return "artifacts.Prepare" } -func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) error { err := InsertPythonArtifact(ctx, b) if err != nil { - return diag.FromErr(err) + return err } for _, artifactName := range slices.Sorted(maps.Keys(b.Config.Artifacts)) { artifact := b.Config.Artifacts[artifactName] if artifact == nil { l := b.Config.GetLocation("artifacts." + artifactName) - logdiag.LogDiag(ctx, diag.Diagnostic{ + return diag.Diagnostic{ Severity: diag.Error, Summary: "Artifact not properly configured", Detail: "please specify artifact properties", Locations: []dyn.Location{l}, - }) - continue + } } b.Metrics.AddBoolValue(metrics.ArtifactBuildCommandIsSet, artifact.BuildCommand != "") b.Metrics.AddBoolValue(metrics.ArtifactFilesIsSet, len(artifact.Files) != 0) @@ -85,15 +83,13 @@ func (m *prepare) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } if artifact.BuildCommand == "" && len(artifact.Files) == 0 { - logdiag.LogError(ctx, errors.New("misconfigured artifact: please specify 'build' or 'files' property")) + return errors.New("misconfigured artifact: please specify 'build' or 'files' property") } if len(artifact.Files) > 0 && artifact.BuildCommand == "" { - bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName}) - } - - if logdiag.HasError(ctx) { - break + if err := bundle.ApplyContext(ctx, b, expandGlobs{name: artifactName}); err != nil { + return err + } } } diff --git a/bundle/artifacts/upload.go b/bundle/artifacts/upload.go index 341c1ee632e..47af7bb5440 100644 --- a/bundle/artifacts/upload.go +++ b/bundle/artifacts/upload.go @@ -20,10 +20,10 @@ func (m *cleanUp) Name() string { return "artifacts.CleanUp" } -func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - client, uploadPath, diags := libraries.GetFilerForLibrariesCleanup(ctx, b) - if diags.HasError() { - return diags +func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) error { + client, uploadPath, err := libraries.GetFilerForLibrariesCleanup(ctx, b) + if err != nil { + return err } skipArtifactsCleanup := b.Config.Experimental != nil && b.Config.Experimental.SkipArtifactCleanup @@ -38,7 +38,7 @@ func (m *cleanUp) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics } } - err := client.Mkdir(ctx, libraries.InternalDirName) + err = client.Mkdir(ctx, libraries.InternalDirName) if err != nil { return diag.Errorf("unable to create directory for %s: %v", uploadPath, err) } diff --git a/bundle/bundle.go b/bundle/bundle.go index a471a5b9b2e..b81d49e32fe 100644 --- a/bundle/bundle.go +++ b/bundle/bundle.go @@ -189,47 +189,33 @@ func Load(ctx context.Context, path string) (*Bundle, error) { } // MustLoad returns a bundle configuration. -// The errors are recorded by logdiag, check with logdiag.HasError(). -func MustLoad(ctx context.Context) *Bundle { +func MustLoad(ctx context.Context) (*Bundle, error) { root, err := mustGetRoot(ctx) if err != nil { - logdiag.LogError(ctx, err) - return nil + return nil, err } logdiag.SetRoot(ctx, root) - b, err := Load(ctx, root) - if err != nil { - logdiag.LogError(ctx, err) - return nil - } - return b + return Load(ctx, root) } // TryLoad returns a bundle configuration if there is one, but doesn't fail if there isn't one. -// The errors are recorded by logdiag, check with logdiag.HasError(). // It returns a `nil` bundle if a bundle was not found. -func TryLoad(ctx context.Context) *Bundle { +func TryLoad(ctx context.Context) (*Bundle, error) { root, err := tryGetRoot(ctx) if err != nil { - logdiag.LogError(ctx, err) - return nil + return nil, err } // No root is fine in this function. if root == "" { - return nil + return nil, nil } logdiag.SetRoot(ctx, root) - b, err := Load(ctx, root) - if err != nil { - logdiag.LogError(ctx, err) - return nil - } - return b + return Load(ctx, root) } func (b *Bundle) initClientOnce(ctx context.Context) { diff --git a/bundle/bundle_test.go b/bundle/bundle_test.go index 37928cb8801..7ba8a4a68ec 100644 --- a/bundle/bundle_test.go +++ b/bundle/bundle_test.go @@ -22,8 +22,11 @@ import ( func mustLoad(t *testing.T) (*Bundle, []diag.Diagnostic) { ctx := logdiag.InitContext(t.Context()) logdiag.SetCollect(ctx, true) - b := MustLoad(ctx) + b, err := MustLoad(ctx) diags := logdiag.FlushCollected(ctx) + if err != nil { + diags = append(diags, diag.DiagnosticFromError(err)) + } return b, diags } @@ -31,8 +34,11 @@ func mustLoad(t *testing.T) (*Bundle, []diag.Diagnostic) { func tryLoad(t *testing.T) (*Bundle, []diag.Diagnostic) { ctx := logdiag.InitContext(t.Context()) logdiag.SetCollect(ctx, true) - b := TryLoad(ctx) + b, err := TryLoad(ctx) diags := logdiag.FlushCollected(ctx) + if err != nil { + diags = append(diags, diag.DiagnosticFromError(err)) + } return b, diags } diff --git a/bundle/config/loader/entry_point.go b/bundle/config/loader/entry_point.go index d476cb221c4..86d641a42eb 100644 --- a/bundle/config/loader/entry_point.go +++ b/bundle/config/loader/entry_point.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) type entryPoint struct{} @@ -19,18 +20,19 @@ func (m *entryPoint) Name() string { return "EntryPoint" } -func (m *entryPoint) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *entryPoint) Apply(ctx context.Context, b *bundle.Bundle) error { path, err := config.FileNames.FindInPath(b.BundleRootPath) if err != nil { - return diag.FromErr(err) + return err } this, diags := config.Load(path) - if diags.HasError() { - return diags + for _, d := range diags { + if d.Severity != diag.Error { + logdiag.LogDiag(ctx, d) + } } - err = b.Config.Merge(this) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) + if err := diags.Error(); err != nil { + return err } - return diags + return b.Config.Merge(this) } diff --git a/bundle/config/loader/process_include.go b/bundle/config/loader/process_include.go index 3a64297814c..da335bb5815 100644 --- a/bundle/config/loader/process_include.go +++ b/bundle/config/loader/process_include.go @@ -11,16 +11,17 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) -func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics { +func validateFileFormat(ctx context.Context, configRoot dyn.Value, filePath string) error { for _, resourceDescription := range config.SupportedResources() { singularName := resourceDescription.SingularName for _, yamlExt := range []string{"yml", "yaml"} { ext := fmt.Sprintf(".%s.%s", singularName, yamlExt) if strings.HasSuffix(filePath, ext) { - return validateSingleResourceDefined(configRoot, ext, singularName) + return validateSingleResourceDefined(ctx, configRoot, ext, singularName) } } } @@ -28,7 +29,7 @@ func validateFileFormat(configRoot dyn.Value, filePath string) diag.Diagnostics return nil } -func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.Diagnostics { +func validateSingleResourceDefined(ctx context.Context, configRoot dyn.Value, ext, typ string) error { type resource struct { path dyn.Path value dyn.Value @@ -53,7 +54,7 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D return v, nil }) if err != nil { - return diag.FromErr(err) + return err } // Gather all resources defined in a target block. @@ -70,7 +71,7 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D return v, nil }) if err != nil { - return diag.FromErr(err) + return err } typeMatch := true @@ -121,15 +122,14 @@ func validateSingleResourceDefined(configRoot dyn.Value, ext, typ string) diag.D return cmp.Compare(a.String(), b.String()) }) - return diag.Diagnostics{ - { - Severity: diag.Recommendation, - Summary: fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext), - Detail: detail.String(), - Locations: locations, - Paths: paths, - }, - } + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Recommendation, + Summary: fmt.Sprintf("define a single %s in a file with the %s extension.", strings.ReplaceAll(typ, "_", " "), ext), + Detail: detail.String(), + Locations: locations, + Paths: paths, + }) + return nil } type processInclude struct { @@ -149,20 +149,24 @@ func (m *processInclude) Name() string { return fmt.Sprintf("ProcessInclude(%s)", m.relPath) } -func (m *processInclude) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *processInclude) Apply(ctx context.Context, b *bundle.Bundle) error { this, diags := config.Load(m.fullPath) - if diags.HasError() { - return diags + for _, d := range diags { + if d.Severity != diag.Error { + logdiag.LogDiag(ctx, d) + } + } + if err := diags.Error(); err != nil { + return err } // Add any diagnostics associated with the file format. - diags = append(diags, validateFileFormat(this.Value(), m.relPath)...) - if diags.HasError() { - return diags + if err := validateFileFormat(ctx, this.Value(), m.relPath); err != nil { + return err } if len(this.Include) > 0 { - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: "Include section is defined outside root file", Detail: `An include section is defined in a file that is not databricks.yml. @@ -172,9 +176,5 @@ Only includes defined in databricks.yml are applied.`, }) } - err := b.Config.Merge(this) - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } - return diags + return b.Config.Merge(this) } diff --git a/bundle/config/loader/process_root_includes.go b/bundle/config/loader/process_root_includes.go index 74803f9c06b..c00c3a29ce5 100644 --- a/bundle/config/loader/process_root_includes.go +++ b/bundle/config/loader/process_root_includes.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) type processRootIncludes struct{} @@ -36,7 +37,7 @@ func hasGlobCharacters(path string) (string, bool) { return "", false } -func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) error { var out []bundle.Mutator // Map with files we've already seen to avoid loading them twice. @@ -60,13 +61,11 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // 1. Change CWD to the bundle root path before calling [filepath.Glob] // 2. Implement our own custom globbing function. We can use [filepath.Match] to do so. if char, ok := hasGlobCharacters(b.BundleRootPath); ok { - diags = diags.Append(diag.Diagnostic{ + return diag.Diagnostic{ Severity: diag.Error, Summary: "Bundle root path contains glob pattern characters", Detail: fmt.Sprintf("The path to the bundle root %s contains glob pattern character %q. Please remove the character from this path to use bundle commands.", b.BundleRootPath, char), - }) - - return diags + } } // For each glob, find all files to load. @@ -81,7 +80,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // Anchor includes to the bundle root path. matches, err := filepath.Glob(filepath.Join(b.BundleRootPath, entry)) if err != nil { - return diag.FromErr(err) + return err } // If the entry is not a glob pattern and no matches found, @@ -95,7 +94,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. for _, match := range matches { rel, err := filepath.Rel(b.BundleRootPath, match) if err != nil { - return diag.FromErr(err) + return err } if _, ok := seen[rel]; ok { continue @@ -115,7 +114,7 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. } if len(diags) > 0 { - return diags + return logdiag.Flush(ctx, diags) } // Add matches to list of mutators to return. @@ -133,6 +132,5 @@ func (m *processRootIncludes) Apply(ctx context.Context, b *bundle.Bundle) diag. // to account for the root databricks.yaml file. b.Metrics.ConfigurationFileCount = int64(len(files)) + 1 - bundle.ApplySeqContext(ctx, b, out...) - return nil + return bundle.ApplySeqContext(ctx, b, out...) } diff --git a/bundle/config/mutator/apply_source_linked_deployment_preset.go b/bundle/config/mutator/apply_source_linked_deployment_preset.go index c1e817673d9..14d2e52996e 100644 --- a/bundle/config/mutator/apply_source_linked_deployment_preset.go +++ b/bundle/config/mutator/apply_source_linked_deployment_preset.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/dbr" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type applySourceLinkedDeploymentPreset struct{} @@ -22,48 +23,43 @@ func (m *applySourceLinkedDeploymentPreset) Name() string { return "ApplySourceLinkedDeploymentPreset" } -func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle.Bundle) error { if config.IsExplicitlyDisabled(b.Config.Presets.SourceLinkedDeployment) { return nil } - var diags diag.Diagnostics isDatabricksWorkspace := dbr.RunsOnRuntime(ctx) && strings.HasPrefix(b.SyncRootPath, "/Workspace/") target := b.Config.Bundle.Target if config.IsExplicitlyEnabled((b.Config.Presets.SourceLinkedDeployment)) { if !isDatabricksWorkspace { path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) - diags = diags.Append( - diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment is available only in the Databricks Workspace", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path[2:].String()), + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "source-linked deployment is available only in the Databricks Workspace", + Paths: []dyn.Path{ + path, }, - ) + Locations: b.Config.GetLocations(path[2:].String()), + }) disabled := false b.Config.Presets.SourceLinkedDeployment = &disabled - return diags + return nil } b.Metrics.AddBoolValue("source_linked_set_for_non_development", b.Config.Bundle.Mode != config.Development) if b.Config.Bundle.Mode != config.Development { path := dyn.NewPath(dyn.Key("targets"), dyn.Key(target), dyn.Key("presets"), dyn.Key("source_linked_deployment")) - diags = diags.Append( - diag.Diagnostic{ - Severity: diag.Warning, - Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path[2:].String()), + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "source-linked deployment in non-development mode is deprecated and will not be supported in a future release", + Paths: []dyn.Path{ + path, }, - ) + Locations: b.Config.GetLocations(path[2:].String()), + }) } } @@ -75,18 +71,16 @@ func (m *applySourceLinkedDeploymentPreset) Apply(ctx context.Context, b *bundle // This mutator runs before workspace paths are defaulted so it's safe to check for the user-defined value if b.Config.Workspace.FilePath != "" && config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { path := dyn.NewPath(dyn.Key("workspace"), dyn.Key("file_path")) - diags = diags.Append( - diag.Diagnostic{ - Severity: diag.Warning, - Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", - Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", - Paths: []dyn.Path{ - path, - }, - Locations: b.Config.GetLocations(path.String()), + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "workspace.file_path setting will be ignored in source-linked deployment mode", + Detail: "In source-linked deployment files are not copied to the destination and resources use source files instead", + Paths: []dyn.Path{ + path, }, - ) + Locations: b.Config.GetLocations(path.String()), + }) } - return diags + return nil } diff --git a/bundle/config/mutator/artifacts_dynamic_version.go b/bundle/config/mutator/artifacts_dynamic_version.go index 9d66a7619b7..998f1c39ab3 100644 --- a/bundle/config/mutator/artifacts_dynamic_version.go +++ b/bundle/config/mutator/artifacts_dynamic_version.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" ) type artifactsUseDynamicVersion struct{} @@ -18,7 +17,7 @@ func (m *artifactsUseDynamicVersion) Name() string { return "ApplyArtifactsDynamicVersion" } -func (m *artifactsUseDynamicVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *artifactsUseDynamicVersion) Apply(ctx context.Context, b *bundle.Bundle) error { if !b.Config.Presets.ArtifactsDynamicVersion { return nil } diff --git a/bundle/config/mutator/collect_escape_telemetry.go b/bundle/config/mutator/collect_escape_telemetry.go index b24922bb5b3..13073b8f98b 100644 --- a/bundle/config/mutator/collect_escape_telemetry.go +++ b/bundle/config/mutator/collect_escape_telemetry.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -21,7 +20,7 @@ func (*collectEscapeTelemetry) Name() string { return "CollectEscapeTelemetry" } -func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) error { var hasDoubleDollarBrace, hasDoubleDollar, hasBackslashDollarBrace, hasBackslashDollar bool _, err := dyn.Walk(b.Config.Value(), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { @@ -46,7 +45,7 @@ func (*collectEscapeTelemetry) Apply(ctx context.Context, b *bundle.Bundle) diag return v, nil }) if err != nil { - return diag.FromErr(err) + return err } if hasDoubleDollarBrace { diff --git a/bundle/config/mutator/compute_id_compat.go b/bundle/config/mutator/compute_id_compat.go index 8f1ff5868e8..3df1bd88245 100644 --- a/bundle/config/mutator/compute_id_compat.go +++ b/bundle/config/mutator/compute_id_compat.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type computeIdToClusterId struct{} @@ -18,40 +19,34 @@ func (m *computeIdToClusterId) Name() string { return "ComputeIdToClusterId" } -func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - +func (m *computeIdToClusterId) Apply(ctx context.Context, b *bundle.Bundle) error { // The "compute_id" key is set; rewrite it to "cluster_id". - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.NewPath(dyn.Key("bundle"))) - diags = diags.Extend(d) + return b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + v, err := rewriteComputeIdToClusterId(ctx, v, dyn.NewPath(dyn.Key("bundle"))) + if err != nil { + return dyn.InvalidValue, err + } // Check if the "compute_id" key is set in any target overrides. return dyn.MapByPattern(v, dyn.NewPattern(dyn.Key("targets"), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) { - v, d := rewriteComputeIdToClusterId(v, dyn.Path{}) - diags = diags.Extend(d) - return v, nil + return rewriteComputeIdToClusterId(ctx, v, dyn.Path{}) }) }) - - diags = diags.Extend(diag.FromErr(err)) - return diags } -func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics +func rewriteComputeIdToClusterId(ctx context.Context, v dyn.Value, p dyn.Path) (dyn.Value, error) { computeIdPath := p.Append(dyn.Key("compute_id")) computeId, err := dyn.GetByPath(v, computeIdPath) // If the "compute_id" key is not set, we don't need to do anything. if err != nil { - return v, nil + return v, nil //nolint:nilerr // missing key is not an error here } if computeId.Kind() == dyn.KindInvalid { return v, nil } - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: "compute_id is deprecated, please use cluster_id instead", Locations: computeId.Locations(), @@ -61,10 +56,10 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn clusterIdPath := p.Append(dyn.Key("cluster_id")) nv, err := dyn.SetByPath(v, clusterIdPath, computeId) if err != nil { - return dyn.InvalidValue, diag.FromErr(err) + return dyn.InvalidValue, err } // Drop the "compute_id" key. - vout, err := dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { + return dyn.Walk(nv, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { switch len(p) { case 0: return v, nil @@ -80,7 +75,4 @@ func rewriteComputeIdToClusterId(v dyn.Value, p dyn.Path) (dyn.Value, diag.Diagn } return v, dyn.ErrSkip }) - - diags = diags.Extend(diag.FromErr(err)) - return vout, diags } diff --git a/bundle/config/mutator/configure_wsfs.go b/bundle/config/mutator/configure_wsfs.go index a93fba9e052..2f8a6eff8b2 100644 --- a/bundle/config/mutator/configure_wsfs.go +++ b/bundle/config/mutator/configure_wsfs.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/dbr" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/vfs" ) @@ -21,7 +20,7 @@ func (m *configureWSFS) Name() string { return "ConfigureWSFS" } -func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) error { root := b.SyncRoot.Native() // The bundle root must be located in /Workspace/ @@ -54,7 +53,7 @@ func (m *configureWSFS) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno return filer.NewReadOnlyWorkspaceFilesExtensionsClient(ctx, b.WorkspaceClient(ctx), path) }) if err != nil { - return diag.FromErr(err) + return err } b.SyncRoot = p diff --git a/bundle/config/mutator/default_target.go b/bundle/config/mutator/default_target.go index 73d99002a06..d5318a3e26c 100644 --- a/bundle/config/mutator/default_target.go +++ b/bundle/config/mutator/default_target.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/diag" ) type defineDefaultTarget struct { @@ -25,7 +24,7 @@ func (m *defineDefaultTarget) Name() string { return fmt.Sprintf("DefineDefaultTarget(%s)", m.name) } -func (m *defineDefaultTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *defineDefaultTarget) Apply(_ context.Context, b *bundle.Bundle) error { // Nothing to do if the configuration has at least 1 target. if len(b.Config.Targets) > 0 { return nil diff --git a/bundle/config/mutator/default_workspace_paths.go b/bundle/config/mutator/default_workspace_paths.go index 02a1ddb3b11..76c68aad567 100644 --- a/bundle/config/mutator/default_workspace_paths.go +++ b/bundle/config/mutator/default_workspace_paths.go @@ -19,7 +19,7 @@ func (m *defineDefaultWorkspacePaths) Name() string { return "DefaultWorkspacePaths" } -func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *defineDefaultWorkspacePaths) Apply(ctx context.Context, b *bundle.Bundle) error { root := b.Config.Workspace.RootPath if root == "" { return diag.Errorf("unable to define default workspace paths: workspace root not defined") diff --git a/bundle/config/mutator/default_workspace_root.go b/bundle/config/mutator/default_workspace_root.go index d7c24a5b557..3de9afc6719 100644 --- a/bundle/config/mutator/default_workspace_root.go +++ b/bundle/config/mutator/default_workspace_root.go @@ -19,7 +19,7 @@ func (m *defineDefaultWorkspaceRoot) Name() string { return "DefineDefaultWorkspaceRoot" } -func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *defineDefaultWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) error { if b.Config.Workspace.RootPath != "" { return nil } diff --git a/bundle/config/mutator/environments_compat.go b/bundle/config/mutator/environments_compat.go index fb898edea36..71c0cc19f84 100644 --- a/bundle/config/mutator/environments_compat.go +++ b/bundle/config/mutator/environments_compat.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -19,7 +18,7 @@ func (m *environmentsToTargets) Name() string { return "EnvironmentsToTargets" } -func (m *environmentsToTargets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *environmentsToTargets) Apply(ctx context.Context, b *bundle.Bundle) error { // Short circuit if the "environments" key is not set. // This is the common case. if b.Config.Environments == nil { @@ -62,5 +61,5 @@ func (m *environmentsToTargets) Apply(ctx context.Context, b *bundle.Bundle) dia return v, nil }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/expand_workspace_root.go b/bundle/config/mutator/expand_workspace_root.go index 2ec70548fed..84671b8671d 100644 --- a/bundle/config/mutator/expand_workspace_root.go +++ b/bundle/config/mutator/expand_workspace_root.go @@ -20,7 +20,7 @@ func (m *expandWorkspaceRoot) Name() string { return "ExpandWorkspaceRoot" } -func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *expandWorkspaceRoot) Apply(ctx context.Context, b *bundle.Bundle) error { root := b.Config.Workspace.RootPath if root == "" { return diag.Errorf("unable to expand workspace root: workspace root not defined") diff --git a/bundle/config/mutator/initialize_cache.go b/bundle/config/mutator/initialize_cache.go index d27e52efc12..f2ba2faab7c 100644 --- a/bundle/config/mutator/initialize_cache.go +++ b/bundle/config/mutator/initialize_cache.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cache" - "github.com/databricks/cli/libs/diag" ) type initializeCache struct{} @@ -20,7 +19,7 @@ func (m *initializeCache) Name() string { return "InitializeCache" } -func (m *initializeCache) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *initializeCache) Apply(ctx context.Context, b *bundle.Bundle) error { // Initialize cache with 30 minute expiry for user information b.Cache = cache.NewCache(ctx, "user", 30*time.Minute, &b.Metrics) return nil diff --git a/bundle/config/mutator/initialize_urls.go b/bundle/config/mutator/initialize_urls.go index c3a877d9e86..6ef5a06404c 100644 --- a/bundle/config/mutator/initialize_urls.go +++ b/bundle/config/mutator/initialize_urls.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/auth" - "github.com/databricks/cli/libs/diag" ) type initializeURLs struct{} @@ -24,15 +23,15 @@ func (m *initializeURLs) Name() string { return "InitializeURLs" } -func (m *initializeURLs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *initializeURLs) Apply(ctx context.Context, b *bundle.Bundle) error { workspaceID, err := auth.ResolveWorkspaceID(ctx, b.WorkspaceClient(ctx)) if err != nil { - return diag.FromErr(err) + return err } host := b.WorkspaceClient(ctx).Config.CanonicalHostName() err = initializeForWorkspace(b, workspaceID, host) if err != nil { - return diag.FromErr(err) + return err } return nil } diff --git a/bundle/config/mutator/initialize_variables.go b/bundle/config/mutator/initialize_variables.go index e72cdde3103..8e50b4d0410 100644 --- a/bundle/config/mutator/initialize_variables.go +++ b/bundle/config/mutator/initialize_variables.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/variable" - "github.com/databricks/cli/libs/diag" ) type initializeVariables struct{} @@ -19,7 +18,7 @@ func (m *initializeVariables) Name() string { return "InitializeVariables" } -func (m *initializeVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *initializeVariables) Apply(ctx context.Context, b *bundle.Bundle) error { vars := b.Config.Variables for k, v := range vars { if v == nil { diff --git a/bundle/config/mutator/load_dbalert_files.go b/bundle/config/mutator/load_dbalert_files.go index 992ce9962d6..d95fb993a46 100644 --- a/bundle/config/mutator/load_dbalert_files.go +++ b/bundle/config/mutator/load_dbalert_files.go @@ -43,7 +43,7 @@ func (d AlertFile) MarshalJSON() ([]byte, error) { return marshal.Marshal(d) } -func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) error { // Fields that are only settable in the API, and are not allowed in .dbalert.json. // We will only allow these fields to be set in the bundle YAML when an .dbalert.json is // specified. This is done to only have one way to set these fields when a .dbalert.json is @@ -57,13 +57,13 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia alertV, err := dyn.GetByPath(b.Config.Value(), dyn.NewPath(dyn.Key("resources"), dyn.Key("alerts"), dyn.Key(alertKey))) if err != nil { - return diag.FromErr(err) + return err } // No other fields other than allowedInYAML should be set in the bundle YAML. m, ok := alertV.AsMap() if !ok { - return diag.FromErr(fmt.Errorf("internal error: alert value is not a map: %w", err)) + return fmt.Errorf("internal error: alert value is not a map: %w", err) } for _, p := range m.Pairs() { @@ -78,58 +78,50 @@ func (m *loadDBAlertFiles) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia continue } - return diag.Diagnostics{ - { - ID: "", - Severity: diag.Error, - Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), - Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))}, - Locations: v.Locations(), - }, + return diag.Diagnostic{ + ID: "", + Severity: diag.Error, + Summary: fmt.Sprintf("field %s is not allowed in the bundle configuration.", k), + Detail: "When a .dbalert.json is specified, only the following fields are allowed in the bundle configuration: " + strings.Join(allowedInYAML, ", "), + Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.%s", alertKey, k))}, + Locations: v.Locations(), } } content, err := os.ReadFile(alert.FilePath) if err != nil { - return diag.Diagnostics{ - { - ID: diag.ID(""), - Severity: diag.Error, - Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), - Detail: "", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, - Locations: alertV.Get("file_path").Locations(), - }, + return diag.Diagnostic{ + ID: diag.ID(""), + Severity: diag.Error, + Summary: fmt.Sprintf("failed to read .dbalert.json file %s: %s", alert.FilePath, err), + Detail: "", + Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Locations: alertV.Get("file_path").Locations(), } } var dbalertFromFile AlertFile err = json.Unmarshal(content, &dbalertFromFile) if err != nil { - return diag.Diagnostics{ - { - ID: diag.ID(""), - Severity: diag.Error, - Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), - Detail: "", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, - Locations: alertV.Get("file_path").Locations(), - }, + return diag.Diagnostic{ + ID: diag.ID(""), + Severity: diag.Error, + Summary: fmt.Sprintf("failed to parse .dbalert.json file %s: %s", alert.FilePath, err), + Detail: "", + Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Locations: alertV.Get("file_path").Locations(), } } // Check that the file does not have any variable interpolations. if dynvar.ContainsVariableReference(string(content)) { - return diag.Diagnostics{ - { - ID: diag.ID(""), - Severity: diag.Error, - Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), - Detail: "Please inline the alert configuration in the bundle configuration to use variables", - Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, - Locations: alertV.Get("file_path").Locations(), - }, + return diag.Diagnostic{ + ID: diag.ID(""), + Severity: diag.Error, + Summary: fmt.Sprintf(".alert file %s must not contain variable interpolations.", alert.FilePath), + Detail: "Please inline the alert configuration in the bundle configuration to use variables", + Paths: []dyn.Path{dyn.MustPathFromString(fmt.Sprintf("resources.alerts.%s.file_path", alertKey))}, + Locations: alertV.Get("file_path").Locations(), } } diff --git a/bundle/config/mutator/load_git_details.go b/bundle/config/mutator/load_git_details.go index f2d3be08a1b..b6d8685c609 100644 --- a/bundle/config/mutator/load_git_details.go +++ b/bundle/config/mutator/load_git_details.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/git" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/vfs" ) @@ -20,11 +21,12 @@ func (m *loadGitDetails) Name() string { return "LoadGitDetails" } -func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics +func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) error { info, err := git.FetchRepositoryInfo(ctx, b.BundleRoot.Native(), b.WorkspaceClient(ctx)) if err != nil { - diags = append(diags, diag.WarningFromErr(err)...) + for _, d := range diag.WarningFromErr(err) { + logdiag.LogDiag(ctx, d) + } } if info.WorktreeRoot == "" { @@ -51,9 +53,8 @@ func (m *loadGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn relBundlePath, err := filepath.Rel(b.WorktreeRoot.Native(), b.BundleRoot.Native()) if err != nil { - diags = append(diags, diag.FromErr(err)...) - } else { - b.Config.Bundle.Git.BundleRootPath = filepath.ToSlash(relBundlePath) + return err } - return diags + b.Config.Bundle.Git.BundleRootPath = filepath.ToSlash(relBundlePath) + return nil } diff --git a/bundle/config/mutator/mutator.go b/bundle/config/mutator/mutator.go index a00488a50fa..90db5f8d165 100644 --- a/bundle/config/mutator/mutator.go +++ b/bundle/config/mutator/mutator.go @@ -10,8 +10,8 @@ import ( "github.com/databricks/cli/bundle/scripts" ) -func DefaultMutators(ctx context.Context, b *bundle.Bundle) { - bundle.ApplySeqContext(ctx, b, +func DefaultMutators(ctx context.Context, b *bundle.Bundle) error { + return bundle.ApplySeqContext(ctx, b, loader.EntryPoint(), // Execute preinit script before processing includes. diff --git a/bundle/config/mutator/normalize_paths.go b/bundle/config/mutator/normalize_paths.go index ecab0e812db..e343e23480e 100644 --- a/bundle/config/mutator/normalize_paths.go +++ b/bundle/config/mutator/normalize_paths.go @@ -13,7 +13,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator/paths" "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -39,7 +38,7 @@ func NormalizePaths() bundle.Mutator { return &normalizePaths{} } -func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) error { // Do not normalize job task paths if using git source gitSourcePaths := collectGitSourcePaths(b) @@ -63,10 +62,10 @@ func (a normalizePaths) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnost }) }) if err != nil { - return diag.FromErr(fmt.Errorf("failed to normalize paths: %w", err)) + return fmt.Errorf("failed to normalize paths: %w", err) } - return diag.FromErr(err) + return err } func collectGitSourcePaths(b *bundle.Bundle) []dyn.Path { diff --git a/bundle/config/mutator/populate_current_user.go b/bundle/config/mutator/populate_current_user.go index b1ec81b33e8..017e67c4bbb 100644 --- a/bundle/config/mutator/populate_current_user.go +++ b/bundle/config/mutator/populate_current_user.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/cache" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/iamutil" "github.com/databricks/cli/libs/tags" "github.com/databricks/databricks-sdk-go/service/iam" @@ -23,7 +22,7 @@ func (m *populateCurrentUser) Name() string { return "PopulateCurrentUser" } -func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) error { if b.Config.Workspace.CurrentUser != nil { return nil } @@ -34,7 +33,7 @@ func (m *populateCurrentUser) Apply(ctx context.Context, b *bundle.Bundle) diag. return w.CurrentUser.Me(ctx, iam.MeRequest{}) }) if err != nil { - return diag.FromErr(err) + return err } b.Config.Workspace.CurrentUser = &config.User{ diff --git a/bundle/config/mutator/populate_locations.go b/bundle/config/mutator/populate_locations.go index 8f7a1258b1e..34b1fd73e87 100644 --- a/bundle/config/mutator/populate_locations.go +++ b/bundle/config/mutator/populate_locations.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn/dynloc" ) @@ -20,14 +19,14 @@ func (m *populateLocations) Name() string { return "PopulateLocations" } -func (m *populateLocations) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *populateLocations) Apply(ctx context.Context, b *bundle.Bundle) error { locs, err := dynloc.Build( b.Config.Value(), // Make all paths relative to the bundle root. b.BundleRootPath, ) if err != nil { - return diag.FromErr(err) + return err } b.Config.Locations = &locs diff --git a/bundle/config/mutator/prepend_workspace_prefix.go b/bundle/config/mutator/prepend_workspace_prefix.go index 3124244c857..06dfa7e8004 100644 --- a/bundle/config/mutator/prepend_workspace_prefix.go +++ b/bundle/config/mutator/prepend_workspace_prefix.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -26,7 +25,7 @@ var skipPrefixes = []string{ "/Volumes/", } -func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) error { patterns := []dyn.Pattern{ dyn.NewPattern(dyn.Key("workspace"), dyn.Key("root_path")), dyn.NewPattern(dyn.Key("workspace"), dyn.Key("file_path")), @@ -65,7 +64,7 @@ func (m *prependWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di return v, nil }) if err != nil { - return diag.FromErr(err) + return err } return nil diff --git a/bundle/config/mutator/python/python_locations_test.go b/bundle/config/mutator/python/python_locations_test.go index 9c5b33d3b59..6481b151645 100644 --- a/bundle/config/mutator/python/python_locations_test.go +++ b/bundle/config/mutator/python/python_locations_test.go @@ -5,7 +5,6 @@ import ( "path/filepath" "testing" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynassert" "github.com/stretchr/testify/assert" @@ -141,13 +140,13 @@ func TestLoadOutput(t *testing.T) { location, ) - value, diags := loadOutput( + value, err := loadOutput( bundleRoot, bytes.NewReader([]byte(output)), locations, ) - assert.Equal(t, diag.Diagnostics{}, diags) + require.NoError(t, err) name, err := dyn.Get(value, "resources.jobs.my_job.name") require.NoError(t, err) diff --git a/bundle/config/mutator/python/python_mutator.go b/bundle/config/mutator/python/python_mutator.go index 9112aaa808d..0c2a0e40de8 100644 --- a/bundle/config/mutator/python/python_mutator.go +++ b/bundle/config/mutator/python/python_mutator.go @@ -198,10 +198,10 @@ func applyBackwardsCompatibilityFixes(b *bundle.Bundle) error { }) } -func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) error { err := applyBackwardsCompatibilityFixes(b) if err != nil { - return diag.FromErr(fmt.Errorf("failed to apply backwards compatibility fixes: %w", err)) + return fmt.Errorf("failed to apply backwards compatibility fixes: %w", err) } opts, err := getOpts(b, m.phase) @@ -224,13 +224,10 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno // re-invokes `databricks auth token --host `. authEnv, err := b.AuthEnv(ctx) if err != nil { - return diag.FromErr(err) + return err } - // mutateDiags is used because Mutate returns 'error' instead of 'diag.Diagnostics' - var mutateDiags diag.Diagnostics var result applyPythonOutputResult - mutateDiagsHasError := errors.New("unexpected error") err = b.Config.Mutate(func(leftRoot dyn.Value) (dyn.Value, error) { pythonPath, err := detectExecutable(ctx, opts.venvPath) @@ -244,16 +241,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno } defer cleanup() - rightRoot, diags := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{ + rightRoot, err := m.runPythonMutator(ctx, leftRoot, runPythonMutatorOpts{ cacheDir: cacheDir, bundleRootPath: b.BundleRootPath, pythonPath: pythonPath, loadLocations: opts.loadLocations, authEnv: authEnv, }) - mutateDiags = diags - if diags.HasError() { - return dyn.InvalidValue, mutateDiagsHasError + if err != nil { + return dyn.InvalidValue, err } newRoot, result0, err := applyPythonOutput(leftRoot, rightRoot) @@ -293,27 +289,15 @@ func (m *pythonMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno b.Metrics.PythonUpdatedResourcesCount += int64(result.UpdatedResources.Size()) b.Metrics.PythonAddedResourcesCount += int64(result.AddedResources.Size()) - if err == mutateDiagsHasError { - if !mutateDiags.HasError() { - panic("mutateDiags has no error, but error is expected") - } - - return mutateDiags - } else { - mutateDiags = mutateDiags.Extend(diag.FromErr(err)) - } - - if mutateDiags.HasError() { - return mutateDiags + if err != nil { + return err } - resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources) - if logdiag.HasError(ctx) { - return mutateDiags + if err := resourcemutator.NormalizeAndInitializeResources(ctx, b, result.AddedResources); err != nil { + return err } - resourcemutator.NormalizeResources(ctx, b, result.UpdatedResources) - return mutateDiags + return resourcemutator.NormalizeResources(ctx, b, result.UpdatedResources) } // createCacheDir returns the directory for input/output files of the Python subprocess, and a cleanup function. @@ -343,7 +327,7 @@ func createCacheDir(ctx context.Context) (string, func(), error) { return cacheDir, func() { _ = os.RemoveAll(cacheDir) }, nil } -func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, opts runPythonMutatorOpts) (dyn.Value, diag.Diagnostics) { +func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, opts runPythonMutatorOpts) (dyn.Value, error) { inputPath := filepath.Join(opts.cacheDir, "input.json") outputPath := filepath.Join(opts.cacheDir, "output.json") diagnosticsPath := filepath.Join(opts.cacheDir, "diagnostics.json") @@ -371,6 +355,14 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op return dyn.InvalidValue, diag.Errorf("failed to write input file: %s", err) } + logPythonWarnings := func(diags diag.Diagnostics) { + for _, d := range diags { + if d.Severity != diag.Error { + logdiag.LogDiag(ctx, d) + } + } + } + stderrBuf := bytes.Buffer{} stderrWriter := io.MultiWriter( newLogWriter(ctx, "stderr: "), @@ -397,37 +389,44 @@ func (m *pythonMutator) runPythonMutator(ctx context.Context, root dyn.Value, op // if diagnostics file exists, it gives the most descriptive errors // if there is any error, we treat it as fatal error, and stop processing - if pythonDiagnostics.HasError() { - return dyn.InvalidValue, pythonDiagnostics + if err := pythonDiagnostics.Error(); err != nil { + logPythonWarnings(pythonDiagnostics) + return dyn.InvalidValue, err } // process can fail without reporting errors in diagnostics file or creating it, for instance, // venv doesn't have 'databricks-bundles' library installed if processErr != nil { - diagnostic := diag.Diagnostic{ + logPythonWarnings(pythonDiagnostics) + return dyn.InvalidValue, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("python mutator process failed: %q, use --debug to enable logging", processErr), Detail: explainProcessErr(ctx, stderrBuf.String()), } - - return dyn.InvalidValue, diag.Diagnostics{diagnostic} } // or we can fail to read diagnostics file, that should always be created if pythonDiagnosticsErr != nil { + logPythonWarnings(pythonDiagnostics) return dyn.InvalidValue, diag.Errorf("failed to load diagnostics: %s", pythonDiagnosticsErr) } locations, err := loadLocationsFile(opts.bundleRootPath, locationsPath) if err != nil { + logPythonWarnings(pythonDiagnostics) return dyn.InvalidValue, diag.Errorf("failed to load locations: %s", err) } - output, outputDiags := loadOutputFile(opts.bundleRootPath, outputPath, locations) - pythonDiagnostics = pythonDiagnostics.Extend(outputDiags) + output, err := loadOutputFile(opts.bundleRootPath, outputPath, locations) + if err != nil { + logPythonWarnings(pythonDiagnostics) + return dyn.InvalidValue, err + } - // we pass through pythonDiagnostic because it contains warnings - return output, pythonDiagnostics + // pythonDiagnostics only contains warnings at this point; surface them. + logPythonWarnings(pythonDiagnostics) + + return output, nil } const pythonInstallExplanation = `Ensure that 'databricks-bundles' is installed in Python environment: @@ -481,10 +480,10 @@ func loadLocationsFile(bundleRoot, locationsPath string) (*pythonLocations, erro return parsePythonLocations(bundleRoot, locationsFile) } -func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { +func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dyn.Value, error) { outputFile, err := os.Open(outputPath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to open output file: %w", err)) + return dyn.InvalidValue, fmt.Errorf("failed to open output file: %w", err) } defer outputFile.Close() @@ -492,7 +491,7 @@ func loadOutputFile(rootPath, outputPath string, locations *pythonLocations) (dy return loadOutput(rootPath, outputFile, locations) } -func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (dyn.Value, diag.Diagnostics) { +func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocations) (dyn.Value, error) { // we need absolute path because later parts of pipeline assume all paths are absolute // and this file will be used as location to resolve relative paths. // @@ -503,41 +502,38 @@ func loadOutput(rootPath string, outputFile io.Reader, locations *pythonLocation // for that, we pass virtualPath instead of outputPath as file location virtualPath, err := filepath.Abs(filepath.Join(rootPath, generatedFileName)) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to get absolute path: %w", err)) + return dyn.InvalidValue, fmt.Errorf("failed to get absolute path: %w", err) } generated, err := yamlloader.LoadYAML(virtualPath, outputFile) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse output file: %w", err)) + return dyn.InvalidValue, fmt.Errorf("failed to parse output file: %w", err) } // generated has dyn.Location as if it comes from generated YAML file // earlier we loaded locations.json with source locations in Python code generatedWithLocations, err := mergePythonLocations(generated, locations) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to update locations: %w", err)) + return dyn.InvalidValue, fmt.Errorf("failed to update locations: %w", err) } return strictNormalize(config.Root{}, generatedWithLocations) } -func strictNormalize(dst any, generated dyn.Value) (dyn.Value, diag.Diagnostics) { +func strictNormalize(dst any, generated dyn.Value) (dyn.Value, error) { normalized, diags := convert.Normalize(dst, generated) // warnings shouldn't happen because output should be already normalized - // when it happens, it's a bug in the mutator, and should be treated as an error - + // when it happens, it's a bug in the mutator, and should be treated as an error. strictDiags := diag.Diagnostics{} - for _, d := range diags { if d.Severity == diag.Warning { d.Severity = diag.Error } - strictDiags = strictDiags.Append(d) } - return normalized, strictDiags + return normalized, strictDiags.Error() } // loadDiagnosticsFile loads diagnostics from a file. diff --git a/bundle/config/mutator/python/python_mutator_test.go b/bundle/config/mutator/python/python_mutator_test.go index 9b106f727bf..f65de7157a7 100644 --- a/bundle/config/mutator/python/python_mutator_test.go +++ b/bundle/config/mutator/python/python_mutator_test.go @@ -466,10 +466,10 @@ func TestStrictNormalize(t *testing.T) { value := dyn.NewValue(map[string]dyn.Value{"A": dyn.NewValue("abc", nil)}, nil) _, diags := convert.Normalize(TestStruct{}, value) - _, strictDiags := strictNormalize(TestStruct{}, value) + _, strictErr := strictNormalize(TestStruct{}, value) assert.False(t, diags.HasError()) - assert.True(t, strictDiags.HasError()) + assert.Error(t, strictErr) } func TestCreateCacheDir(t *testing.T) { diff --git a/bundle/config/mutator/resolve_lookup_variables.go b/bundle/config/mutator/resolve_lookup_variables.go index e642997eedf..6c3af51afbd 100644 --- a/bundle/config/mutator/resolve_lookup_variables.go +++ b/bundle/config/mutator/resolve_lookup_variables.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "golang.org/x/sync/errgroup" ) @@ -16,7 +15,7 @@ func ResolveLookupVariables() bundle.Mutator { return &resolveLookupVariables{} } -func (m *resolveLookupVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *resolveLookupVariables) Apply(ctx context.Context, b *bundle.Bundle) error { errs, errCtx := errgroup.WithContext(ctx) for k := range b.Config.Variables { @@ -41,7 +40,7 @@ func (m *resolveLookupVariables) Apply(ctx context.Context, b *bundle.Bundle) di } // Note, diags are lost from all goroutines except the first one to return diag - return diag.FromErr(errs.Wait()) + return errs.Wait() } func (*resolveLookupVariables) Name() string { diff --git a/bundle/config/mutator/resolve_select.go b/bundle/config/mutator/resolve_select.go index 28d9751c322..12cd4317efb 100644 --- a/bundle/config/mutator/resolve_select.go +++ b/bundle/config/mutator/resolve_select.go @@ -25,7 +25,7 @@ func (m *resolveSelect) Name() string { return "ResolveSelect" } -func (m *resolveSelect) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *resolveSelect) Apply(_ context.Context, b *bundle.Bundle) error { if len(b.Select) == 0 { return nil } diff --git a/bundle/config/mutator/resolve_variable_references.go b/bundle/config/mutator/resolve_variable_references.go index 113f0576394..55270486ec7 100644 --- a/bundle/config/mutator/resolve_variable_references.go +++ b/bundle/config/mutator/resolve_variable_references.go @@ -15,6 +15,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/logdiag" ) /* @@ -141,7 +142,7 @@ func (m *resolveVariableReferences) Validate(ctx context.Context, b *bundle.Bund return nil } -func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) error { prefixes := make([]dyn.Path, len(m.prefixes)) for i, prefix := range m.prefixes { prefixes[i] = dyn.MustPathFromString(prefix) @@ -151,16 +152,12 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) // We rewrite it here to make the resolution logic simpler. varPath := dyn.NewPath(dyn.Key("var")) - var diags diag.Diagnostics maxRounds := 1 + m.extraRounds for round := range maxRounds { - hasUpdates, newDiags := m.resolveOnce(b, prefixes, varPath) - - diags = diags.Extend(newDiags) - - if diags.HasError() { - break + hasUpdates, err := m.resolveOnce(ctx, b, prefixes, varPath) + if err != nil { + return err } if !hasUpdates { @@ -168,7 +165,7 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) } if round >= maxRounds-1 { - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("Variables references are too deep, stopping resolution after %d rounds. Unresolved variables may remain.", round+1), // Would be nice to include names of the variables there, but that would complicate things more @@ -181,11 +178,10 @@ func (m *resolveVariableReferences) Apply(ctx context.Context, b *bundle.Bundle) b.Metrics.SetBoolValue("artifacts_reference_used", true) } - return diags + return nil } -func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn.Path, varPath dyn.Path) (bool, diag.Diagnostics) { - var diags diag.Diagnostics +func (m *resolveVariableReferences) resolveOnce(ctx context.Context, b *bundle.Bundle, prefixes []dyn.Path, varPath dyn.Path) (bool, error) { hasUpdates := false err := m.selectivelyMutate(b, func(root dyn.Value) (dyn.Value, error) { // Synthesize a copy of the root that has all fields that are present in the type @@ -244,14 +240,19 @@ func (m *resolveVariableReferences) resolveOnce(b *bundle.Bundle, prefixes []dyn // Normalize the result because variable resolution may have been applied to non-string fields. // For example, a variable reference may have been resolved to a integer. root, normaliseDiags := convert.Normalize(b.Config, root) - diags = diags.Extend(normaliseDiags) + for _, d := range normaliseDiags { + logdiag.LogDiag(ctx, d) + } + if err := normaliseDiags.Error(); err != nil { + return dyn.InvalidValue, err + } return root, nil }) if err != nil { - diags = diags.Extend(diag.FromErr(err)) + return hasUpdates, err } - return hasUpdates, diags + return hasUpdates, nil } // selectivelyMutate applies a function to a subset of the configuration diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go index cbb05d7d622..4c45a8c0250 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions.go @@ -97,10 +97,10 @@ func ApplyBundlePermissions() bundle.Mutator { return &bundlePermissions{} } -func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) error { err := validatePermissions(b) if err != nil { - return diag.FromErr(err) + return err } patterns := make(map[string]dyn.Pattern, 0) @@ -154,7 +154,7 @@ func (m *bundlePermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Di return v, nil }) if err != nil { - return diag.FromErr(err) + return err } return nil diff --git a/bundle/config/mutator/resourcemutator/apply_presets.go b/bundle/config/mutator/resourcemutator/apply_presets.go index 70cc2f66cbc..f9cba7d4140 100644 --- a/bundle/config/mutator/resourcemutator/apply_presets.go +++ b/bundle/config/mutator/resourcemutator/apply_presets.go @@ -36,11 +36,9 @@ func (m *applyPresets) Name() string { return "ApplyPresets" } -func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - - if d := validatePauseStatus(b); d != nil { - diags = diags.Extend(d) +func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) error { + if err := validatePauseStatus(b); err != nil { + return err } r := b.Config.Resources @@ -310,19 +308,19 @@ func (m *applyPresets) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos // it as the deployment id), so prefixing would change the resource's // identity rather than just its display name. - return diags + return nil } -func validatePauseStatus(b *bundle.Bundle) diag.Diagnostics { +func validatePauseStatus(b *bundle.Bundle) error { p := b.Config.Presets.TriggerPauseStatus if p == "" || p == config.Paused || p == config.Unpaused { return nil } - return diag.Diagnostics{{ + return diag.Diagnostic{ Summary: "Invalid value for trigger_pause_status, should be PAUSED or UNPAUSED", Severity: diag.Error, Locations: []dyn.Location{b.Config.GetLocation("presets.trigger_pause_status")}, - }} + } } // toTagArray converts a map of tags to an array of tags. diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode.go b/bundle/config/mutator/resourcemutator/apply_target_mode.go index 727a2e5bef0..f749e54c7cf 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode.go @@ -5,7 +5,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" ) @@ -61,7 +60,7 @@ func transformDevelopmentMode(ctx context.Context, b *bundle.Bundle) { } } -func (m *applyTargetMode) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *applyTargetMode) Apply(ctx context.Context, b *bundle.Bundle) error { if b.Config.Bundle.Mode == config.Development { transformDevelopmentMode(ctx, b) } diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index 61c2fed2592..a4e979efd3b 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" ) type captureUCDependencies struct{} @@ -88,7 +87,7 @@ func resolveCatalog(b *bundle.Bundle, catalogName string) string { return catalogName } -func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) error { // Resolve resources that depend on schemas before resolving schemas themselves. // The schema resolution below modifies schema.CatalogName, and findSchema // (used by resolveSchema) matches against the original schema.CatalogName value. diff --git a/bundle/config/mutator/resourcemutator/cluster_fixups.go b/bundle/config/mutator/resourcemutator/cluster_fixups.go index 893cd248aa4..0d6f7dd6c15 100644 --- a/bundle/config/mutator/resourcemutator/cluster_fixups.go +++ b/bundle/config/mutator/resourcemutator/cluster_fixups.go @@ -5,7 +5,6 @@ import ( "slices" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -20,7 +19,7 @@ func (m *jobClustersFixups) Name() string { return "JobClustersFixups" } -func (m *jobClustersFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *jobClustersFixups) Apply(ctx context.Context, b *bundle.Bundle) error { for _, job := range b.Config.Resources.Jobs { if job == nil { continue @@ -44,7 +43,7 @@ func (m *clusterFixups) Name() string { return "ClusterFixups" } -func (m *clusterFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *clusterFixups) Apply(ctx context.Context, b *bundle.Bundle) error { for _, cluster := range b.Config.Resources.Clusters { if cluster == nil { continue diff --git a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go index 27b55403d88..804e90baa6f 100644 --- a/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go +++ b/bundle/config/mutator/resourcemutator/configure_dashboards_serialized_dashboard.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -24,9 +23,7 @@ func (c configureDashboardSerializedDashboard) Name() string { return "ConfigureDashboardSerializedDashboard" } -func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - +func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundle.Bundle) error { pattern := dyn.NewPattern( dyn.Key("resources"), dyn.Key("dashboards"), @@ -53,6 +50,5 @@ func (c configureDashboardSerializedDashboard) Apply(_ context.Context, b *bundl }) }) - diags = diags.Extend(diag.FromErr(err)) - return diags + return err } diff --git a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go index c51c97a51e4..b5865a6e334 100644 --- a/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go +++ b/bundle/config/mutator/resourcemutator/configure_genie_space_serialized_space.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) const serializedSpaceFieldName = "serialized_space" @@ -22,7 +23,7 @@ func (c configureGenieSpaceSerializedSpace) Name() string { return "ConfigureGenieSpaceSerializedSpace" } -func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (c configureGenieSpaceSerializedSpace) Apply(ctx context.Context, b *bundle.Bundle) error { var diags diag.Diagnostics pattern := dyn.NewPattern( @@ -81,7 +82,9 @@ func (c configureGenieSpaceSerializedSpace) Apply(_ context.Context, b *bundle.B } }) }) + if err != nil { + return err + } - diags = diags.Extend(diag.FromErr(err)) - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/mutator/resourcemutator/dashboard_fixups.go b/bundle/config/mutator/resourcemutator/dashboard_fixups.go index 5dbb67a174a..6e743030df4 100644 --- a/bundle/config/mutator/resourcemutator/dashboard_fixups.go +++ b/bundle/config/mutator/resourcemutator/dashboard_fixups.go @@ -6,7 +6,6 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" ) type dashboardFixups struct{} @@ -28,7 +27,7 @@ func ensureWorkspacePrefix(parentPath string) string { return path.Join("/Workspace", parentPath) } -func (m *dashboardFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *dashboardFixups) Apply(ctx context.Context, b *bundle.Bundle) error { for _, dashboard := range b.Config.Resources.Dashboards { if dashboard == nil { continue diff --git a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go index 38f835793ea..3b213dcee78 100644 --- a/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go +++ b/bundle/config/mutator/resourcemutator/expand_pipeline_glob_paths.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/patchwheel" ) @@ -91,7 +90,7 @@ func (m *expandPipelineGlobPaths) expandSequence(ctx context.Context, dir string return dyn.NewValue(vs, v.Locations()), nil } -func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { p := dyn.NewPattern( dyn.Key("resources"), @@ -106,7 +105,7 @@ func (m *expandPipelineGlobPaths) Apply(ctx context.Context, b *bundle.Bundle) d }) }) - return diag.FromErr(err) + return err } func (*expandPipelineGlobPaths) Name() string { diff --git a/bundle/config/mutator/resourcemutator/fix_permissions.go b/bundle/config/mutator/resourcemutator/fix_permissions.go index 7f685015e2d..fdc26a397e3 100644 --- a/bundle/config/mutator/resourcemutator/fix_permissions.go +++ b/bundle/config/mutator/resourcemutator/fix_permissions.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" ) @@ -210,7 +209,7 @@ func createPermissionFromPrincipal(principal, level string) dyn.Value { return dyn.V(permission) } -func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) error { currentUser := b.Config.Workspace.CurrentUser.UserName err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { @@ -223,5 +222,5 @@ func (m *fixPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn ), processPermissions(currentUser)) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/resourcemutator/genie_space_fixups.go b/bundle/config/mutator/resourcemutator/genie_space_fixups.go index 85e1bb7e745..9b9b2a2b8cc 100644 --- a/bundle/config/mutator/resourcemutator/genie_space_fixups.go +++ b/bundle/config/mutator/resourcemutator/genie_space_fixups.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" ) type genieSpaceFixups struct{} @@ -17,7 +16,7 @@ func (m *genieSpaceFixups) Name() string { return "GenieSpaceFixups" } -func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *genieSpaceFixups) Apply(ctx context.Context, b *bundle.Bundle) error { for _, genieSpace := range b.Config.Resources.GenieSpaces { if genieSpace == nil { continue diff --git a/bundle/config/mutator/resourcemutator/merge_apps.go b/bundle/config/mutator/resourcemutator/merge_apps.go index edbdf7712ce..7de92df875a 100644 --- a/bundle/config/mutator/resourcemutator/merge_apps.go +++ b/bundle/config/mutator/resourcemutator/merge_apps.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" ) @@ -30,7 +29,7 @@ func (m *mergeApps) resourceName(v dyn.Value) string { } } -func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { if v.Kind() == dyn.KindNil { return v, nil @@ -41,5 +40,5 @@ func (m *mergeApps) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic })) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/resourcemutator/merge_grants.go b/bundle/config/mutator/resourcemutator/merge_grants.go index a90e9a57507..6e51d54559a 100644 --- a/bundle/config/mutator/resourcemutator/merge_grants.go +++ b/bundle/config/mutator/resourcemutator/merge_grants.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" ) @@ -31,7 +30,7 @@ func (m *mergeGrants) Name() string { return "MergeGrants" } -func (m *mergeGrants) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *mergeGrants) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { if v.Kind() == dyn.KindNil { return v, nil @@ -63,7 +62,7 @@ func (m *mergeGrants) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return v, nil }) - return diag.FromErr(err) + return err } // deduplicateSequence removes duplicate values from a dyn sequence, diff --git a/bundle/config/mutator/resourcemutator/merge_job_clusters.go b/bundle/config/mutator/resourcemutator/merge_job_clusters.go index c7c46a3fdef..ad9a8cb5406 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_clusters.go +++ b/bundle/config/mutator/resourcemutator/merge_job_clusters.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" ) @@ -30,7 +29,7 @@ func (m *mergeJobClusters) jobClusterKey(v dyn.Value) string { } } -func (m *mergeJobClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *mergeJobClusters) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { if v.Kind() == dyn.KindNil { return v, nil @@ -41,5 +40,5 @@ func (m *mergeJobClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia })) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/resourcemutator/merge_job_parameters.go b/bundle/config/mutator/resourcemutator/merge_job_parameters.go index 79458b36a60..c8343811e0c 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_parameters.go +++ b/bundle/config/mutator/resourcemutator/merge_job_parameters.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" ) @@ -30,7 +29,7 @@ func (m *mergeJobParameters) parameterNameString(v dyn.Value) string { } } -func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { if v.Kind() == dyn.KindNil { return v, nil @@ -41,5 +40,5 @@ func (m *mergeJobParameters) Apply(ctx context.Context, b *bundle.Bundle) diag.D })) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/resourcemutator/merge_job_tasks.go b/bundle/config/mutator/resourcemutator/merge_job_tasks.go index b85a863d80f..037522bb980 100644 --- a/bundle/config/mutator/resourcemutator/merge_job_tasks.go +++ b/bundle/config/mutator/resourcemutator/merge_job_tasks.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" ) @@ -30,7 +29,7 @@ func (m *mergeJobTasks) taskKeyString(v dyn.Value) string { } } -func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { if v.Kind() == dyn.KindNil { return v, nil @@ -45,5 +44,5 @@ func (m *mergeJobTasks) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno })) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go b/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go index 6496a00d90e..a284b588089 100644 --- a/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go +++ b/bundle/config/mutator/resourcemutator/merge_pipeline_clusters.go @@ -5,7 +5,6 @@ import ( "strings" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/merge" ) @@ -33,7 +32,7 @@ func (m *mergePipelineClusters) clusterLabel(v dyn.Value) string { } } -func (m *mergePipelineClusters) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *mergePipelineClusters) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { if v.Kind() == dyn.KindNil { return v, nil @@ -44,5 +43,5 @@ func (m *mergePipelineClusters) Apply(ctx context.Context, b *bundle.Bundle) dia })) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go b/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go index 87962a85e86..ed0b83e5246 100644 --- a/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go +++ b/bundle/config/mutator/resourcemutator/model_serving_endpoint_fixups.go @@ -6,6 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/utils" "github.com/databricks/databricks-sdk-go/service/serving" ) @@ -42,7 +43,7 @@ func servedModelToServedEntity(model serving.ServedModelInput) serving.ServedEnt } } -func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle) error { var diags diag.Diagnostics for key, endpoint := range b.Config.Resources.ModelServingEndpoints { @@ -67,7 +68,7 @@ func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle // We perform this translation here so that the deployment plan only has to detect served_entities and can ignore served_models. if len(endpoint.Config.ServedModels) > 0 { // Add warning recommending served_entities - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: "Using served_models is deprecated", Detail: "The served_models field is deprecated. Please use served_entities instead.", @@ -85,5 +86,5 @@ func (m *modelServingEndpointFixups) Apply(ctx context.Context, b *bundle.Bundle } } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/mutator/resourcemutator/override_compute.go b/bundle/config/mutator/resourcemutator/override_compute.go index 6b5251248d5..f7ad32fa704 100644 --- a/bundle/config/mutator/resourcemutator/override_compute.go +++ b/bundle/config/mutator/resourcemutator/override_compute.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/logdiag" ) type overrideCompute struct{} @@ -38,17 +39,15 @@ func overrideJobCompute(j *resources.Job, compute string) { } } -func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - +func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) error { if b.Config.Bundle.Mode == config.Production { if b.Config.Bundle.ClusterId != "" { // Overriding compute via a command-line flag for production works, but is not recommended. - diags = diags.Extend(diag.Diagnostics{{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Summary: "Setting a cluster override for a target that uses 'mode: production' is not recommended", Detail: "It is recommended to always use the same compute for production target for consistency.", Severity: diag.Warning, - }}) + }) } } if v := env.Get(ctx, "DATABRICKS_CLUSTER_ID"); v != "" { @@ -58,13 +57,17 @@ func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag cmdio.LogString(ctx, "Setting a cluster override because DATABRICKS_CLUSTER_ID is set. It is recommended to use --cluster-id instead, which works in any target mode.") } else { // We don't allow using DATABRICKS_CLUSTER_ID in any other mode, it's too error-prone. - return diag.Warningf("The DATABRICKS_CLUSTER_ID variable is set but is ignored since the current target does not use 'mode: development'") + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "The DATABRICKS_CLUSTER_ID variable is set but is ignored since the current target does not use 'mode: development'", + }) + return nil } b.Config.Bundle.ClusterId = v } if b.Config.Bundle.ClusterId == "" { - return diags + return nil } r := b.Config.Resources @@ -72,5 +75,5 @@ func (m *overrideCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag overrideJobCompute(r.Jobs[i], b.Config.Bundle.ClusterId) } - return diags + return nil } diff --git a/bundle/config/mutator/resourcemutator/process_static_resources.go b/bundle/config/mutator/resourcemutator/process_static_resources.go index 7d3ad742e4b..bb8f5124ae3 100644 --- a/bundle/config/mutator/resourcemutator/process_static_resources.go +++ b/bundle/config/mutator/resourcemutator/process_static_resources.go @@ -6,9 +6,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/logdiag" ) type processStaticResources struct{} @@ -28,17 +26,17 @@ func (p processStaticResources) Name() string { return "ProcessStaticResources" } -func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) error { addedResources, err := getAllResources(b) if err != nil { - return diag.FromErr(err) + return err } // only YAML resources need to have paths normalized, before normalizing paths // we need to resolve variables because they can change path values: // - variable can be used a prefix // - path can be part of a complex variable value - bundle.ApplySeqContext( + err = bundle.ApplySeqContext( ctx, b, // Reads (dynamic): * (strings) (searches for variable references in string values) @@ -52,13 +50,11 @@ func (p processStaticResources) Apply(ctx context.Context, b *bundle.Bundle) dia // since the latter reads dashboard files and requires fully resolved paths. mutator.TranslatePathsDashboards(), ) - - if logdiag.HasError(ctx) { - return nil + if err != nil { + return err } - NormalizeAndInitializeResources(ctx, b, addedResources) - return nil + return NormalizeAndInitializeResources(ctx, b, addedResources) } func getAllResources(b *bundle.Bundle) (ResourceKeySet, error) { diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 45740f53599..2fb04542c13 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/libs/dyn/merge" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/dyn" @@ -19,8 +18,8 @@ import ( // settings and defaults to it. Initialization is applied only once. // // If bundle is modified outside of 'resources' section, these changes are discarded. -func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { - bundle.ApplySeqContext( +func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) error { + err := bundle.ApplySeqContext( ctx, b, // Reads (typed): b.Config.RunAs, b.Config.Workspace.CurrentUser (validates run_as configuration) @@ -41,9 +40,8 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // ApplyPresets should have more priority than defaults below, so it should be run first ApplyPresets(), ) - - if logdiag.HasError(ctx) { - return + if err != nil { + return err } defaults := []struct { @@ -104,13 +102,12 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { } for _, defaultDef := range defaults { - bundle.SetDefault(ctx, b, defaultDef.pattern, defaultDef.value) - if logdiag.HasError(ctx) { - return + if err := bundle.SetDefault(ctx, b, defaultDef.pattern, defaultDef.value); err != nil { + return err } } - bundle.ApplySeqContext(ctx, b, + return bundle.ApplySeqContext(ctx, b, // Reads (typed): b.Config.Resources.Dashboards (checks dashboard configurations) // Updates (typed): b.Config.Resources.Dashboards[].ParentPath (ensures /Workspace prefix is present) // Ensures dashboard parent paths have the required /Workspace prefix @@ -136,8 +133,8 @@ func applyInitializeMutators(ctx context.Context, b *bundle.Bundle) { // Normalization is applied multiple times if resource is modified during initialization // // If bundle is modified outside of 'resources' section, these changes are discarded. -func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { - bundle.ApplySeqContext( +func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) error { + return bundle.ApplySeqContext( ctx, b, @@ -215,9 +212,9 @@ func NormalizeAndInitializeResources( ctx context.Context, b *bundle.Bundle, addedResources ResourceKeySet, -) { +) error { if addedResources.IsEmpty() { - return + return nil } var snapshot dyn.Value @@ -228,18 +225,15 @@ func NormalizeAndInitializeResources( return selectResources(root, addedResources) }) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to select resources: %s", err)) - return + return fmt.Errorf("failed to select resources: %w", err) } - applyNormalizeMutators(ctx, b) - if logdiag.HasError(ctx) { - return + if err := applyNormalizeMutators(ctx, b); err != nil { + return err } - applyInitializeMutators(ctx, b) - if logdiag.HasError(ctx) { - return + if err := applyInitializeMutators(ctx, b); err != nil { + return err } // after mutators, we merge updated resources back to snapshot to preserve non-selected resources @@ -247,8 +241,10 @@ func NormalizeAndInitializeResources( return mergeResources(root, snapshot) }) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %s", err)) + return fmt.Errorf("failed to merge resources: %w", err) } + + return nil } // NormalizeResources normalizes resources specified resources, @@ -257,9 +253,9 @@ func NormalizeResources( ctx context.Context, b *bundle.Bundle, updatedResources ResourceKeySet, -) { +) error { if updatedResources.IsEmpty() { - return + return nil } var snapshot dyn.Value @@ -270,13 +266,11 @@ func NormalizeResources( return selectResources(root, updatedResources) }) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to select resources: %s", err)) - return + return fmt.Errorf("failed to select resources: %w", err) } - applyNormalizeMutators(ctx, b) - if logdiag.HasError(ctx) { - return + if err := applyNormalizeMutators(ctx, b); err != nil { + return err } // after mutators, we merge updated resources back to snapshot to preserve non-selected resources @@ -284,8 +278,10 @@ func NormalizeResources( return mergeResources(root, snapshot) }) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("failed to merge resources: %s", err)) + return fmt.Errorf("failed to merge resources: %w", err) } + + return nil } // selectResources returns bundle configuration with resources only present in resourcePaths. diff --git a/bundle/config/mutator/resourcemutator/run_as.go b/bundle/config/mutator/resourcemutator/run_as.go index 4f5e3ce9036..f21ce02f03f 100644 --- a/bundle/config/mutator/resourcemutator/run_as.go +++ b/bundle/config/mutator/resourcemutator/run_as.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/service/jobs" "github.com/databricks/databricks-sdk-go/service/pipelines" "github.com/databricks/databricks-sdk-go/service/sql" @@ -213,7 +214,7 @@ func setPipelineOwnersToRunAsIdentity(b *bundle.Bundle) { } } -func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *setRunAs) Apply(ctx context.Context, b *bundle.Bundle) error { // Track the use of the legacy run_as mode. b.Metrics.AddBoolValue("experimental.use_legacy_run_as", b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs) @@ -230,20 +231,18 @@ func (m *setRunAs) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { if b.Config.Experimental != nil && b.Config.Experimental.UseLegacyRunAs { setPipelineOwnersToRunAsIdentity(b) setRunAsForJobs(b) - return diag.Diagnostics{ - { - Severity: diag.Warning, - Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", - Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, - Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), - }, - } + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "You are using the legacy mode of run_as. The support for this mode is experimental and might be removed in a future release of the CLI. In order to run the pipelines in your DABs project as the run_as user this mode changes the owners of the pipelines to the run_as identity, which requires the user deploying the bundle to be a workspace admin, and also a Metastore admin if the pipeline target is in UC.", + Paths: []dyn.Path{dyn.MustPathFromString("experimental.use_legacy_run_as")}, + Locations: b.Config.GetLocations("experimental.use_legacy_run_as"), + }) + return nil } // Assert the run_as configuration is valid in the context of the bundle - diags := validateRunAs(b) - if diags.HasError() { - return diags + if err := validateRunAs(b).Error(); err != nil { + return err } setRunAsForJobs(b) diff --git a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go index e85584da3f0..eeb41045471 100644 --- a/bundle/config/mutator/resourcemutator/secret_scope_fixups.go +++ b/bundle/config/mutator/resourcemutator/secret_scope_fixups.go @@ -120,7 +120,7 @@ func collapsePermissions(scope *resources.SecretScope) error { return nil } -func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) error { // Secret scopes by default have the current user as a MANAGE ACL. We need to add it to the client ACL list // to prevent a phantom persistent diff. // We do not need to do this in terraform because terraform naively always applies the config during ACL @@ -142,14 +142,12 @@ func (m *secretScopeFixups) Apply(ctx context.Context, b *bundle.Bundle) diag.Di addManageForCurrentUser(scope, currentUser) err := collapsePermissions(scope) if err != nil { - return diag.Diagnostics{ - { - Severity: diag.Error, - Summary: "Failed to collapse permissions for secret scope", - Detail: err.Error(), - Paths: []dyn.Path{dyn.MustPathFromString("resources.secret_scopes." + key)}, - Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, - }, + return diag.Diagnostic{ + Severity: diag.Error, + Summary: "Failed to collapse permissions for secret scope", + Detail: err.Error(), + Paths: []dyn.Path{dyn.MustPathFromString("resources.secret_scopes." + key)}, + Locations: []dyn.Location{b.Config.GetLocation("resources.secret_scopes." + key)}, } } } diff --git a/bundle/config/mutator/resourcemutator/validate_target_mode.go b/bundle/config/mutator/resourcemutator/validate_target_mode.go index 190c60c9b98..8db3f6e2fe9 100644 --- a/bundle/config/mutator/resourcemutator/validate_target_mode.go +++ b/bundle/config/mutator/resourcemutator/validate_target_mode.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) type validateTargetMode struct{} @@ -25,13 +26,13 @@ func (v validateTargetMode) Name() string { return "ValidateTargetMode" } -func (v validateTargetMode) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v validateTargetMode) Apply(ctx context.Context, b *bundle.Bundle) error { switch b.Config.Bundle.Mode { case config.Development: - return validateDevelopmentMode(b) + return logNonErrorsAndReturnError(ctx, validateDevelopmentMode(b)) case config.Production: isPrincipal := iamutil.IsServicePrincipal(b.Config.Workspace.CurrentUser.User) - return validateProductionMode(b, isPrincipal) + return logNonErrorsAndReturnError(ctx, validateProductionMode(b, isPrincipal)) case "": // No action return nil @@ -40,6 +41,17 @@ func (v validateTargetMode) Apply(_ context.Context, b *bundle.Bundle) diag.Diag } } +// logNonErrorsAndReturnError emits warnings and recommendations immediately and +// returns the first error (if any) so the caller can abort the pipeline. +func logNonErrorsAndReturnError(ctx context.Context, diags diag.Diagnostics) error { + for _, d := range diags { + if d.Severity != diag.Error { + logdiag.LogDiag(ctx, d) + } + } + return diags.Error() +} + func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { var diags diag.Diagnostics p := b.Config.Presets @@ -63,10 +75,10 @@ func validateDevelopmentMode(b *bundle.Bundle) diag.Diagnostics { if path := findNonUserPath(b); path != "" { if path == "artifact_path" && strings.HasPrefix(b.Config.Workspace.ArtifactPath, "/Volumes") { // For Volumes paths we recommend including the current username as a substring - diags = diags.Extend(diag.Errorf("%s should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", path)) + diags = diags.Append(diag.DiagnosticFromError(diag.Errorf("%s should contain the current username or ${workspace.current_user.short_name} to ensure uniqueness when using 'mode: development'", path))) } else { // For non-Volumes paths recommend simply putting things in the home folder - diags = diags.Extend(diag.Errorf("%s must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development'", path)) + diags = diags.Append(diag.DiagnosticFromError(diag.Errorf("%s must start with '~/' or contain the current username to ensure uniqueness when using 'mode: development'", path))) } } if p.NamePrefix != "" && !namePrefixContainsUserIdentifier(p.NamePrefix, u) { @@ -129,7 +141,7 @@ func validateProductionMode(b *bundle.Bundle, isPrincipalUsed bool) diag.Diagnos r := b.Config.Resources for i := range r.Pipelines { if r.Pipelines[i].Development { - return diag.Errorf("target with 'mode: production' cannot include a pipeline with 'development: true'") + return diag.Diagnostics{diag.DiagnosticFromError(diag.Errorf("target with 'mode: production' cannot include a pipeline with 'development: true'"))} } } diff --git a/bundle/config/mutator/rewrite_sync_paths.go b/bundle/config/mutator/rewrite_sync_paths.go index 228c5484def..91bad4f41fd 100644 --- a/bundle/config/mutator/rewrite_sync_paths.go +++ b/bundle/config/mutator/rewrite_sync_paths.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -43,7 +42,7 @@ func (m *rewriteSyncPaths) makeRelativeTo(root string) dyn.MapFunc { } } -func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "sync", func(_ dyn.Path, v dyn.Value) (nv dyn.Value, err error) { v, err = dyn.Map(v, "paths", dyn.Foreach(m.makeRelativeTo(b.BundleRootPath))) @@ -89,5 +88,5 @@ func (m *rewriteSyncPaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia }) }) - return diag.FromErr(err) + return err } diff --git a/bundle/config/mutator/rewrite_workspace_prefix.go b/bundle/config/mutator/rewrite_workspace_prefix.go index e66482f8e55..ea40a19d779 100644 --- a/bundle/config/mutator/rewrite_workspace_prefix.go +++ b/bundle/config/mutator/rewrite_workspace_prefix.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type rewriteWorkspacePrefix struct{} @@ -22,8 +23,7 @@ func (m *rewriteWorkspacePrefix) Name() string { return "RewriteWorkspacePrefix" } -func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := diag.Diagnostics{} +func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) error { paths := map[string]string{ "/Workspace/${workspace.root_path}": "${workspace.root_path}", "/Workspace${workspace.root_path}": "${workspace.root_path}", @@ -47,7 +47,7 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di for path, replacePath := range paths { if strings.Contains(vv, path) { newPath := strings.Replace(vv, path, replacePath, 1) - diags = append(diags, diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("substring %q found in %q. Please update this to %q.", path, vv, newPath), Detail: "For more information, please refer to: https://docs.databricks.com/en/release-notes/dev-tools/bundles.html#workspace-paths", @@ -64,8 +64,8 @@ func (m *rewriteWorkspacePrefix) Apply(ctx context.Context, b *bundle.Bundle) di }) }) if err != nil { - return diag.FromErr(err) + return err } - return diags + return nil } diff --git a/bundle/config/mutator/select_default_target.go b/bundle/config/mutator/select_default_target.go index ad8132a46aa..e7f785f7027 100644 --- a/bundle/config/mutator/select_default_target.go +++ b/bundle/config/mutator/select_default_target.go @@ -21,7 +21,7 @@ func (m *selectDefaultTarget) Name() string { return "SelectDefaultTarget" } -func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) error { if len(b.Config.Targets) == 0 { return diag.Errorf("no targets defined") } @@ -29,8 +29,7 @@ func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) diag. // One target means there's only one default. names := slices.Collect(maps.Keys(b.Config.Targets)) if len(names) == 1 { - bundle.ApplyContext(ctx, b, SelectTarget(names[0])) - return nil + return bundle.ApplyContext(ctx, b, SelectTarget(names[0])) } // Multiple targets means we look for the `default` flag. @@ -52,6 +51,5 @@ func (m *selectDefaultTarget) Apply(ctx context.Context, b *bundle.Bundle) diag. } // One default remaining. - bundle.ApplyContext(ctx, b, SelectTarget(defaults[0])) - return nil + return bundle.ApplyContext(ctx, b, SelectTarget(defaults[0])) } diff --git a/bundle/config/mutator/select_target.go b/bundle/config/mutator/select_target.go index 43764a9ee38..e7f349b013a 100644 --- a/bundle/config/mutator/select_target.go +++ b/bundle/config/mutator/select_target.go @@ -27,7 +27,7 @@ func (m *selectTarget) Name() string { return fmt.Sprintf("SelectTarget(%s)", m.name) } -func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) error { if b.Config.Targets == nil { return diag.Errorf("no targets defined") } @@ -41,7 +41,7 @@ func (m *selectTarget) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnosti // Merge specified target into root configuration structure. err := b.Config.MergeTargetOverrides(m.name) if err != nil { - return diag.FromErr(fmt.Errorf("failed to perform target override for target=%s: %w", m.name, err)) + return fmt.Errorf("failed to perform target override for target=%s: %w", m.name, err) } // Store specified target in configuration for reference. diff --git a/bundle/config/mutator/set_variables.go b/bundle/config/mutator/set_variables.go index 3038b05acaf..32d7ddb419e 100644 --- a/bundle/config/mutator/set_variables.go +++ b/bundle/config/mutator/set_variables.go @@ -94,41 +94,39 @@ func setVariable(ctx context.Context, v dyn.Value, variable *variable.Variable, return dyn.InvalidValue, fmt.Errorf(`no value assigned to required variable %s. Variables are usually assigned in databricks.yml, and they can be overridden using "--var", the %s environment variable, or %s`, name, bundleVarPrefix+name, getDefaultVariableFilePath("")) } -func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, diag.Diagnostics) { - var diags diag.Diagnostics - +func readVariablesFromFile(b *bundle.Bundle) (dyn.Value, error) { filePath := filepath.Join(b.BundleRootPath, getDefaultVariableFilePath(b.Config.Bundle.Target)) if _, err := os.Stat(filePath); err != nil { - return dyn.InvalidValue, nil + return dyn.InvalidValue, nil //nolint:nilerr // missing variable file is not an error } f, err := os.ReadFile(filePath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to read variables file: %w", err)) + return dyn.InvalidValue, fmt.Errorf("failed to read variables file: %w", err) } val, err := jsonloader.LoadJSON(f, filePath) if err != nil { - return dyn.InvalidValue, diag.FromErr(fmt.Errorf("failed to parse variables file %s: %w", filePath, err)) + return dyn.InvalidValue, fmt.Errorf("failed to parse variables file %s: %w", filePath, err) } if val.Kind() != dyn.KindMap { - return dyn.InvalidValue, diags.Append(diag.Diagnostic{ + return dyn.InvalidValue, diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("failed to parse variables file %s: invalid format", filePath), Detail: "Variables file must be a JSON object with the following format:\n{\"var1\": \"value1\", \"var2\": \"value2\"}", - }) + } } return val, nil } -func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - defaults, diags := readVariablesFromFile(b) - if diags.HasError() { - return diags +func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) error { + defaults, err := readVariablesFromFile(b) + if err != nil { + return err } - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { + err = b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.Map(v, "variables", dyn.Foreach(func(p dyn.Path, variable dyn.Value) (dyn.Value, error) { name := p[1].Key() v, ok := b.Config.Variables[name] @@ -141,5 +139,5 @@ func (m *setVariables) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnos })) }) - return diags.Extend(diag.FromErr(err)) + return err } diff --git a/bundle/config/mutator/sync_default_path.go b/bundle/config/mutator/sync_default_path.go index 16d4a4d6179..dfe05fbfc2f 100644 --- a/bundle/config/mutator/sync_default_path.go +++ b/bundle/config/mutator/sync_default_path.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -19,7 +18,7 @@ func (m *syncDefaultPath) Name() string { return "SyncDefaultPath" } -func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) error { isset := false err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { pv, _ := dyn.Get(v, "sync.paths") @@ -33,7 +32,7 @@ func (m *syncDefaultPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag return v, nil }) if err != nil { - return diag.FromErr(err) + return err } // If the sync paths field is already set, do nothing. diff --git a/bundle/config/mutator/sync_default_path_test.go b/bundle/config/mutator/sync_default_path_test.go index c4bd6a65af7..39a71f4ad62 100644 --- a/bundle/config/mutator/sync_default_path_test.go +++ b/bundle/config/mutator/sync_default_path_test.go @@ -57,7 +57,7 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) { ctx := logdiag.InitContext(t.Context()) - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { + err := bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { v, err := dyn.Set(v, "sync", dyn.V(dyn.NewMapping())) if err != nil { @@ -71,7 +71,7 @@ func TestSyncDefaultPath_SkipIfSet(t *testing.T) { }) require.NoError(t, err) }) - require.False(t, logdiag.HasError(ctx)) + require.NoError(t, err) diags := bundle.Apply(ctx, b, mutator.SyncDefaultPath()) require.NoError(t, diags.Error()) diff --git a/bundle/config/mutator/sync_infer_root.go b/bundle/config/mutator/sync_infer_root.go index 373a3701dcc..1896720d6ef 100644 --- a/bundle/config/mutator/sync_infer_root.go +++ b/bundle/config/mutator/sync_infer_root.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/vfs" ) @@ -53,9 +54,7 @@ func (m *syncInferRoot) computeRoot(path, root string) string { return filepath.Clean(root) } -func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - +func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) error { // Use the bundle root path as the starting point for inferring the sync root path. bundleRootPath := filepath.Clean(b.BundleRootPath) @@ -78,13 +77,14 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno // Compute the relative path from the sync root to the bundle root. rel, err := filepath.Rel(syncRootPath, bundleRootPath) if err != nil { - return diag.FromErr(err) + return err } // If during computation of the sync root path we hit the root of the filesystem, // then one or more of the sync paths are outside the filesystem. // Check if this happened by verifying that none of the paths escape the root // when joined with the sync root path. + var diags diag.Diagnostics for i, path := range b.Config.Sync.Paths { if filepath.IsLocal(filepath.Join(rel, path)) { continue @@ -99,7 +99,7 @@ func (m *syncInferRoot) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagno } if diags.HasError() { - return diags + return logdiag.Flush(ctx, diags) } // Update all paths in the sync configuration to be relative to the sync root. diff --git a/bundle/config/mutator/translate_paths.go b/bundle/config/mutator/translate_paths.go index b36ec094447..450c27b219a 100644 --- a/bundle/config/mutator/translate_paths.go +++ b/bundle/config/mutator/translate_paths.go @@ -16,8 +16,8 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/notebook" ) @@ -100,6 +100,11 @@ type translateContext struct { // is still needed to produce fully resolved paths for comparison with remote state, // but local file validation would incorrectly fail. skipLocalFileValidation bool + + // errorLogged is set when a fallback path error was logged via logdiag. + // Translation continues so that every offending path is reported, and the + // mutator then fails once all translations have run. + errorLogged bool } // rewritePath converts a given relative path from the loaded config to a new path based on the passed rewriting function @@ -319,7 +324,7 @@ func (t *translateContext) rewriteValue(ctx context.Context, p dyn.Path, v dyn.V return dyn.NewValue(out, v.Locations()), nil } -func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) diag.Diagnostics { +func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContext, translations []func(context.Context, dyn.Value) (dyn.Value, error)) error { // Set the remote root to the sync root if source-linked deployment is enabled. // Otherwise, set it to the workspace file path. if config.IsExplicitlyEnabled(t.b.Config.Presets.SourceLinkedDeployment) { @@ -339,17 +344,17 @@ func applyTranslations(ctx context.Context, b *bundle.Bundle, t *translateContex return v, nil }) - return diag.FromErr(err) + return err } -func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) error { t := &translateContext{ b: b, seen: make(map[seenKey]string), skipLocalFileValidation: b.SkipLocalFileValidation, } - return applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ + err := applyTranslations(ctx, b, t, []func(context.Context, dyn.Value) (dyn.Value, error){ t.applyJobTranslations(paths.VisitJobPaths, false), t.applyJobTranslations(paths.VisitJobLibrariesPaths, true), t.applyPipelineTranslations(paths.VisitPipelinePaths, false), @@ -357,9 +362,19 @@ func (m *translatePaths) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn t.applyArtifactTranslations, t.applyAppsTranslations, }) + if err != nil { + return err + } + + // Fallback path errors are logged (so every offending path is reported) but + // translation continues; fail here once all of them have been surfaced. + if t.errorLogged { + return logdiag.ErrAlreadyPrinted + } + return nil } -func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *translatePathsDashboards) Apply(ctx context.Context, b *bundle.Bundle) error { t := &translateContext{ b: b, seen: make(map[seenKey]string), diff --git a/bundle/config/mutator/translate_paths_jobs.go b/bundle/config/mutator/translate_paths_jobs.go index 4ffab62c89b..7d47c9de48f 100644 --- a/bundle/config/mutator/translate_paths_jobs.go +++ b/bundle/config/mutator/translate_paths_jobs.go @@ -80,6 +80,7 @@ func (t *translateContext) applyJobTranslations(visitor visitFunc, allowOutsideS Summary: fmt.Sprintf("path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).", originalPath, fallback[key], v.Location()), Locations: v.Locations(), }) + t.errorLogged = true return nv, nil } } diff --git a/bundle/config/mutator/translate_paths_pipelines.go b/bundle/config/mutator/translate_paths_pipelines.go index 53848461bd2..8f7ef466976 100644 --- a/bundle/config/mutator/translate_paths_pipelines.go +++ b/bundle/config/mutator/translate_paths_pipelines.go @@ -59,6 +59,7 @@ func (t *translateContext) applyPipelineTranslations(visitor visitFunc, allowOut Summary: fmt.Sprintf("path %s is defined relative to the %s directory (%s). Please update the path to be relative to the file where it is defined or use earlier version of CLI (0.261.0 or earlier).", originalPath, fallback[key], v.Location()), Locations: v.Locations(), }) + t.errorLogged = true return nv, nil } } diff --git a/bundle/config/mutator/translate_paths_test.go b/bundle/config/mutator/translate_paths_test.go index 226a848b723..95047f183a4 100644 --- a/bundle/config/mutator/translate_paths_test.go +++ b/bundle/config/mutator/translate_paths_test.go @@ -853,13 +853,13 @@ func TestTranslatePathWithComplexVariables(t *testing.T) { ctx := t.Context() // Assign the variables to the dynamic configuration. - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { + require.NoError(t, bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { p := dyn.MustPathFromString("resources.jobs.job.tasks[0]") return dyn.SetByPath(v, p.Append(dyn.Key("libraries")), dyn.V("${var.cluster_libraries}")) }) require.NoError(t, err) - }) + })) diags := bundle.ApplySeq(ctx, b, mutator.SetVariables(), diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go index a76feb9482f..71750c2e052 100644 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ b/bundle/config/mutator/validate_direct_only_resources.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) type validateDirectOnlyResources struct { @@ -34,7 +35,7 @@ func isDirectOnly(pluralName string) bool { return hasDirect && !hasTerraform } -func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundle) error { if m.engine.IsDirect() { return nil } @@ -58,5 +59,5 @@ func (m *validateDirectOnlyResources) Apply(ctx context.Context, b *bundle.Bundl }) } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/mutator/validate_git_details.go b/bundle/config/mutator/validate_git_details.go index bb25a198093..43be9c02a0b 100644 --- a/bundle/config/mutator/validate_git_details.go +++ b/bundle/config/mutator/validate_git_details.go @@ -18,7 +18,7 @@ func (m *validateGitDetails) Name() string { return "ValidateGitDetails" } -func (m *validateGitDetails) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *validateGitDetails) Apply(ctx context.Context, b *bundle.Bundle) error { if b.Config.Bundle.Git.Branch == "" || b.Config.Bundle.Git.ActualBranch == "" { return nil } diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go index 67c58a5701c..0fa94268f0f 100644 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ b/bundle/config/mutator/validate_lifecycle_started.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) type validateLifecycleStarted struct { @@ -24,7 +25,7 @@ func (m *validateLifecycleStarted) Name() string { return "ValidateLifecycleStarted" } -func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *validateLifecycleStarted) Apply(ctx context.Context, b *bundle.Bundle) error { if m.engine.IsDirect() { return nil } @@ -44,5 +45,5 @@ func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) di }) } } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/mutator/verify_cli_version.go b/bundle/config/mutator/verify_cli_version.go index 873e4f78065..3886fecfaa8 100644 --- a/bundle/config/mutator/verify_cli_version.go +++ b/bundle/config/mutator/verify_cli_version.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) func VerifyCliVersion() bundle.Mutator { @@ -17,7 +18,7 @@ func VerifyCliVersion() bundle.Mutator { type verifyCliVersion struct{} -func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) error { // No constraints specified, skip the check. if b.Config.Bundle.DatabricksCliVersion == "" { return nil @@ -25,12 +26,12 @@ func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia constraint := b.Config.Bundle.DatabricksCliVersion if err := validateConstraintSyntax(constraint); err != nil { - return diag.FromErr(err) + return err } currentVersion := build.GetInfo().Version c, err := semver.NewConstraint(constraint) if err != nil { - return diag.FromErr(err) + return err } version, err := semver.NewVersion(currentVersion) @@ -40,7 +41,11 @@ func (v *verifyCliVersion) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia if !c.Check(version) { if version.Prerelease() == "dev" && version.Major() == 0 { - return diag.Warningf("Ignoring Databricks CLI version constraint for development build. Required: %s, current: %s", constraint, currentVersion) + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: fmt.Sprintf("Ignoring Databricks CLI version constraint for development build. Required: %s, current: %s", constraint, currentVersion), + }) + return nil } return diag.Errorf("Databricks CLI version constraint not satisfied. Required: %s, current: %s", constraint, currentVersion) diff --git a/bundle/config/root.go b/bundle/config/root.go index caca8e1f1ad..354fae95509 100644 --- a/bundle/config/root.go +++ b/bundle/config/root.go @@ -114,13 +114,19 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { Locations: []dyn.Location{le.Loc}, }} } - return nil, diag.Errorf("failed to load %s: %v", path, err) + return nil, diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to load %s: %v", path, err), + }} } // Rewrite configuration tree where necessary. v, err = rewriteShorthands(v) if err != nil { - return nil, diag.Errorf("failed to rewrite %s: %v", path, err) + return nil, diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to rewrite %s: %v", path, err), + }} } // Normalize dynamic configuration tree according to configuration type. @@ -129,7 +135,10 @@ func LoadFromBytes(path string, raw []byte) (*Root, diag.Diagnostics) { // Convert normalized configuration tree to typed configuration. err = r.updateWithDynamicValue(v) if err != nil { - diags = diags.Extend(diag.Errorf("failed to load %s: %v", path, err)) + diags = diags.Append(diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("failed to load %s: %v", path, err), + }) return nil, diags } return &r, diags diff --git a/bundle/config/validate/all_resources_have_values.go b/bundle/config/validate/all_resources_have_values.go index 7f96e529a74..39909a86411 100644 --- a/bundle/config/validate/all_resources_have_values.go +++ b/bundle/config/validate/all_resources_have_values.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) func AllResourcesHaveValues() bundle.Mutator { @@ -21,8 +22,8 @@ func (m *allResourcesHaveValues) Name() string { return "validate:AllResourcesHaveValues" } -func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := diag.Diagnostics{} +func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) error { + var diags diag.Diagnostics _, err := dyn.MapByPattern( b.Config.Value(), @@ -39,7 +40,7 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di // Name of the resource. Eg: "foo" in "jobs.foo". rName := p[2].Key() - diags = append(diags, diag.Diagnostic{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("%s %s is not defined", rType, rName), Locations: v.Locations(), @@ -50,8 +51,8 @@ func (m *allResourcesHaveValues) Apply(ctx context.Context, b *bundle.Bundle) di }, ) if err != nil { - diags = append(diags, diag.FromErr(err)...) + return err } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/validate/enum.go b/bundle/config/validate/enum.go index e6266163bc8..9af577b8084 100644 --- a/bundle/config/validate/enum.go +++ b/bundle/config/validate/enum.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/logdiag" ) type enum struct{} @@ -23,7 +24,7 @@ func (f *enum) Name() string { return "validate:enum" } -func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) error { diags := diag.Diagnostics{} // Generate prefix tree for all enum fields. @@ -31,12 +32,12 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { for k := range generated.EnumFields { pattern, err := dyn.NewPatternFromString(k) if err != nil { - return diag.FromErr(fmt.Errorf("invalid pattern %q for enum field validation: %w", k, err)) + return fmt.Errorf("invalid pattern %q for enum field validation: %w", k, err) } err = trie.Insert(pattern) if err != nil { - return diag.FromErr(fmt.Errorf("failed to insert pattern %q into trie: %w", k, err)) + return fmt.Errorf("failed to insert pattern %q into trie: %w", k, err) } } @@ -82,7 +83,7 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return nil }) if err != nil { - return diag.FromErr(err) + return err } // Sort diagnostics to make them deterministic @@ -96,5 +97,9 @@ func (f *enum) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return cmp.Compare(fmt.Sprintf("%v", a.Locations), fmt.Sprintf("%v", b.Locations)) }) - return diags + for _, d := range diags { + logdiag.LogDiag(ctx, d) + } + + return nil } diff --git a/bundle/config/validate/fast_validate.go b/bundle/config/validate/fast_validate.go index d01eb8c1491..94b2b6db74a 100644 --- a/bundle/config/validate/fast_validate.go +++ b/bundle/config/validate/fast_validate.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" ) // FastValidate runs a subset of fast validation checks. This is a subset of the full @@ -24,8 +23,8 @@ func (f *fastValidate) Name() string { return "fast_validate(readonly)" } -func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagnostics { - bundle.ApplyParallel(ctx, rb, +func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) error { + return bundle.ApplyParallel(ctx, rb, // Fast mutators with only in-memory checks JobClusterKeyDefined(), JobTaskClusterSpec(), @@ -33,6 +32,4 @@ func (f *fastValidate) Apply(ctx context.Context, rb *bundle.Bundle) diag.Diagno // Blocking mutators. Deployments will fail if these checks fail. ValidateArtifactPath(), ) - - return nil } diff --git a/bundle/config/validate/files_to_sync.go b/bundle/config/validate/files_to_sync.go index aea78f7104b..31d52043ae4 100644 --- a/bundle/config/validate/files_to_sync.go +++ b/bundle/config/validate/files_to_sync.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle/deploy/files" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) func FilesToSync() bundle.ReadOnlyMutator { @@ -19,7 +20,7 @@ func (v *filesToSync) Name() string { return "validate:files_to_sync" } -func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) error { // The user may be intentional about not synchronizing any files. // In this case, we should not show any warnings. if len(b.Config.Sync.Paths) == 0 { @@ -28,12 +29,12 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost sync, err := files.GetSync(ctx, b) if err != nil { - return diag.FromErr(err) + return err } fl, err := sync.GetFileList(ctx) if err != nil { - return diag.FromErr(err) + return err } // If there are files to sync, we don't need to show any warnings. @@ -41,15 +42,14 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost return nil } - diags := diag.Diagnostics{} if len(b.Config.Sync.Exclude) == 0 { - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore", }) } else { path := "sync.exclude" - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: "There are no files to sync, please check your .gitignore and sync.exclude configuration", // Show all locations where sync.exclude is defined, since merging @@ -59,5 +59,5 @@ func (v *filesToSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost }) } - return diags + return nil } diff --git a/bundle/config/validate/files_to_sync_test.go b/bundle/config/validate/files_to_sync_test.go index 71a1dfaf5a7..ffdc2a00bac 100644 --- a/bundle/config/validate/files_to_sync_test.go +++ b/bundle/config/validate/files_to_sync_test.go @@ -1,6 +1,7 @@ package validate import ( + "context" "path/filepath" "testing" @@ -8,6 +9,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/vfs" sdkconfig "github.com/databricks/databricks-sdk-go/config" "github.com/databricks/databricks-sdk-go/experimental/mocks" @@ -18,6 +20,22 @@ import ( "github.com/stretchr/testify/require" ) +// applyFilesToSync runs the read-only mutator directly (without the typed<->dynamic +// config sync that bundle.Apply performs) and returns the diagnostics it logged plus +// the returned error. The sync round-trip would drop the typed-only CurrentUser field +// the test sets, which would force an unexpected workspace API call. +func applyFilesToSync(ctx context.Context, t *testing.T, b *bundle.Bundle) diag.Diagnostics { + t.Helper() + ctx = logdiag.InitContext(ctx) + logdiag.SetCollect(ctx, true) + err := FilesToSync().Apply(ctx, b) + diags := logdiag.FlushCollected(ctx) + if err != nil { + diags = append(diags, diag.DiagnosticFromError(err)) + } + return diags +} + func TestFilesToSync_NoPaths(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{ @@ -28,7 +46,7 @@ func TestFilesToSync_NoPaths(t *testing.T) { } ctx := t.Context() - diags := FilesToSync().Apply(ctx, b) + diags := applyFilesToSync(ctx, t, b) assert.Empty(t, diags) } @@ -83,7 +101,7 @@ func TestFilesToSync_EverythingIgnored(t *testing.T) { testutil.WriteFile(t, filepath.Join(b.BundleRootPath, ".gitignore"), "*\n.*\n") ctx := t.Context() - diags := FilesToSync().Apply(ctx, b) + diags := applyFilesToSync(ctx, t, b) require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore", diags[0].Summary) @@ -96,7 +114,7 @@ func TestFilesToSync_EverythingExcluded(t *testing.T) { b.Config.Sync.Exclude = []string{"*"} ctx := t.Context() - diags := FilesToSync().Apply(ctx, b) + diags := applyFilesToSync(ctx, t, b) require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Equal(t, "There are no files to sync, please check your .gitignore and sync.exclude configuration", diags[0].Summary) diff --git a/bundle/config/validate/folder_permissions.go b/bundle/config/validate/folder_permissions.go index c4355abaf86..30be10ab233 100644 --- a/bundle/config/validate/folder_permissions.go +++ b/bundle/config/validate/folder_permissions.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/bundle/paths" "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/workspace" "golang.org/x/sync/errgroup" @@ -18,45 +19,50 @@ import ( type folderPermissions struct{ bundle.RO } -func (f *folderPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (f *folderPermissions) Apply(ctx context.Context, b *bundle.Bundle) error { if len(b.Config.Permissions) == 0 { return nil } bundlePaths := paths.CollectUniqueWorkspacePathPrefixes(b.Config.Workspace).Paths - var diags diag.Diagnostics g, ctx := errgroup.WithContext(ctx) results := make([]diag.Diagnostics, len(bundlePaths)) for i, p := range bundlePaths { g.Go(func() error { - results[i] = checkFolderPermission(ctx, b, p) + diags, err := checkFolderPermission(ctx, b, p) + if err != nil { + return err + } + results[i] = diags return nil }) } + // Note, only error from first coroutine is captured, others are lost if err := g.Wait(); err != nil { - // Note, only diag from first coroutine is captured, others are lost - diags = diags.Extend(diag.FromErr(err)) + return err } for _, r := range results { - diags = diags.Extend(r) + for _, d := range r { + logdiag.LogDiag(ctx, d) + } } - return diags + return nil } -func checkFolderPermission(ctx context.Context, b *bundle.Bundle, folderPath string) diag.Diagnostics { +func checkFolderPermission(ctx context.Context, b *bundle.Bundle, folderPath string) (diag.Diagnostics, error) { // If the folder is shared, then we don't need to check permissions as it was already checked in the other mutator before. if libraries.IsWorkspaceSharedPath(folderPath) { - return nil + return nil, nil } w := b.WorkspaceClient(ctx).Workspace obj, err := getClosestExistingObject(ctx, w, folderPath) if err != nil { - return diag.FromErr(err) + return nil, err } objPermissions, err := w.GetPermissions(ctx, workspace.GetWorkspaceObjectPermissionsRequest{ @@ -64,11 +70,11 @@ func checkFolderPermission(ctx context.Context, b *bundle.Bundle, folderPath str WorkspaceObjectType: "directories", }) if err != nil { - return diag.FromErr(err) + return nil, err } p := permissions.ObjectAclToResourcePermissions(folderPath, objPermissions.AccessControlList) - return p.Compare(b.Config.Permissions) + return p.Compare(b.Config.Permissions), nil } func getClosestExistingObject(ctx context.Context, w workspace.WorkspaceInterface, folderPath string) (*workspace.ObjectInfo, error) { diff --git a/bundle/config/validate/folder_permissions_test.go b/bundle/config/validate/folder_permissions_test.go index 0cc97907c62..87218f0f55d 100644 --- a/bundle/config/validate/folder_permissions_test.go +++ b/bundle/config/validate/folder_permissions_test.go @@ -68,7 +68,7 @@ func TestFolderPermissionsInheritedWhenRootPathDoesNotExist(t *testing.T) { }, nil) b.SetWorkpaceClient(m.WorkspaceClient) - diags := ValidateFolderPermissions().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateFolderPermissions()) require.Empty(t, diags) } @@ -115,7 +115,7 @@ func TestValidateFolderPermissionsFailsOnMissingBundlePermission(t *testing.T) { }, nil) b.SetWorkpaceClient(m.WorkspaceClient) - diags := ValidateFolderPermissions().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateFolderPermissions()) require.Len(t, diags, 1) require.Equal(t, "workspace folder has permissions not configured in bundle", diags[0].Summary) require.Equal(t, diag.Warning, diags[0].Severity) @@ -163,7 +163,7 @@ func TestValidateFolderPermissionsFailsOnPermissionMismatch(t *testing.T) { }, nil) b.SetWorkpaceClient(m.WorkspaceClient) - diags := ValidateFolderPermissions().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateFolderPermissions()) require.Len(t, diags, 1) require.Equal(t, "workspace folder has permissions not configured in bundle", diags[0].Summary) require.Equal(t, diag.Warning, diags[0].Severity) @@ -196,7 +196,7 @@ func TestValidateFolderPermissionsFailsOnNoRootFolder(t *testing.T) { }) b.SetWorkpaceClient(m.WorkspaceClient) - diags := ValidateFolderPermissions().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateFolderPermissions()) require.Len(t, diags, 1) require.Equal(t, "folder / and its parent folders do not exist", diags[0].Summary) require.Equal(t, diag.Error, diags[0].Severity) diff --git a/bundle/config/validate/interpolation_in_auth_config.go b/bundle/config/validate/interpolation_in_auth_config.go index 1598ef49d88..56a5ace7065 100644 --- a/bundle/config/validate/interpolation_in_auth_config.go +++ b/bundle/config/validate/interpolation_in_auth_config.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/logdiag" ) type noInterpolationInAuthConfig struct{} @@ -21,7 +22,7 @@ func (f *noInterpolationInAuthConfig) Name() string { return "validate:interpolation_in_auth_config" } -func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundle) error { authFields := []string{ // Generic attributes. "host", @@ -48,8 +49,6 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl "workspace_id", } - diags := diag.Diagnostics{} - for _, fieldName := range authFields { p := dyn.NewPath(dyn.Key("workspace"), dyn.Key(fieldName)) v, err := dyn.GetByPath(b.Config.Value(), p) @@ -57,7 +56,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl continue } if err != nil { - return diag.FromErr(err) + return err } vv, ok := v.AsString() @@ -72,7 +71,7 @@ func (f *noInterpolationInAuthConfig) Apply(ctx context.Context, b *bundle.Bundl continue } - diags = append(diags, diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: "Variable interpolation is not supported for fields that configure authentication", Detail: fmt.Sprintf(`Interpolation is not supported for the field %s. Please set @@ -83,5 +82,5 @@ the %s environment variable if you wish to configure this field at runtime.`, p. } } - return diags + return nil } diff --git a/bundle/config/validate/job_cluster_key_defined.go b/bundle/config/validate/job_cluster_key_defined.go index 5ae2f5437b8..d33dddee57f 100644 --- a/bundle/config/validate/job_cluster_key_defined.go +++ b/bundle/config/validate/job_cluster_key_defined.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) func JobClusterKeyDefined() bundle.ReadOnlyMutator { @@ -19,9 +20,7 @@ func (v *jobClusterKeyDefined) Name() string { return "validate:job_cluster_key_defined" } -func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := diag.Diagnostics{} - +func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) error { for k, job := range b.Config.Resources.Jobs { jobClusterKeys := make(map[string]bool) for _, cluster := range job.JobClusters { @@ -35,7 +34,7 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag if _, ok := jobClusterKeys[task.JobClusterKey]; !ok { path := fmt.Sprintf("resources.jobs.%s.tasks[%d].job_cluster_key", k, index) - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("job_cluster_key %s is not defined", task.JobClusterKey), // Show only the location where the job_cluster_key is defined. @@ -49,5 +48,5 @@ func (v *jobClusterKeyDefined) Apply(ctx context.Context, b *bundle.Bundle) diag } } - return diags + return nil } diff --git a/bundle/config/validate/job_cluster_key_defined_test.go b/bundle/config/validate/job_cluster_key_defined_test.go index 4cc35c1ad68..404c5f326a1 100644 --- a/bundle/config/validate/job_cluster_key_defined_test.go +++ b/bundle/config/validate/job_cluster_key_defined_test.go @@ -32,7 +32,7 @@ func TestJobClusterKeyDefined(t *testing.T) { }, } - diags := JobClusterKeyDefined().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, JobClusterKeyDefined()) require.Empty(t, diags) require.NoError(t, diags.Error()) } @@ -55,7 +55,7 @@ func TestJobClusterKeyNotDefined(t *testing.T) { }, } - diags := JobClusterKeyDefined().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) require.Equal(t, diag.Warning, diags[0].Severity) @@ -88,7 +88,7 @@ func TestJobClusterKeyDefinedInDifferentJob(t *testing.T) { }, } - diags := JobClusterKeyDefined().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, JobClusterKeyDefined()) require.Len(t, diags, 1) require.NoError(t, diags.Error()) require.Equal(t, diag.Warning, diags[0].Severity) diff --git a/bundle/config/validate/job_task_cluster_spec.go b/bundle/config/validate/job_task_cluster_spec.go index 79672be63e6..8e514dc47ec 100644 --- a/bundle/config/validate/job_task_cluster_spec.go +++ b/bundle/config/validate/job_task_cluster_spec.go @@ -23,7 +23,7 @@ func (v *jobTaskClusterSpec) Name() string { return "validate:job_task_cluster_spec" } -func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) error { diags := diag.Diagnostics{} jobsPath := dyn.NewPath(dyn.Key("resources"), dyn.Key("jobs")) @@ -38,7 +38,7 @@ func (v *jobTaskClusterSpec) Apply(ctx context.Context, b *bundle.Bundle) diag.D } } - return diags + return diags.Error() } func validateJobTask(b *bundle.Bundle, task jobs.Task, taskPath dyn.Path) diag.Diagnostics { diff --git a/bundle/config/validate/job_task_cluster_spec_test.go b/bundle/config/validate/job_task_cluster_spec_test.go index 3ba2e841a65..90bfb0a7379 100644 --- a/bundle/config/validate/job_task_cluster_spec_test.go +++ b/bundle/config/validate/job_task_cluster_spec_test.go @@ -173,7 +173,7 @@ Specify one of the following fields: job_cluster_key, environment_key, existing_ } b := createBundle(map[string]*resources.Job{"job1": job}) - diags := JobTaskClusterSpec().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, JobTaskClusterSpec()) if tc.errorPath != "" || tc.errorDetail != "" || tc.errorSummary != "" { assert.Len(t, diags, 1) diff --git a/bundle/config/validate/no_interpolation_in_bundle_name.go b/bundle/config/validate/no_interpolation_in_bundle_name.go index c4238ffe25a..e7faeeae0b3 100644 --- a/bundle/config/validate/no_interpolation_in_bundle_name.go +++ b/bundle/config/validate/no_interpolation_in_bundle_name.go @@ -20,7 +20,7 @@ func (m *noInterpolationInBundleName) Name() string { return "validate:no_interpolation_in_bundle_name" } -func (m *noInterpolationInBundleName) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *noInterpolationInBundleName) Apply(ctx context.Context, b *bundle.Bundle) error { if dynvar.ContainsVariableReference(b.Config.Bundle.Name) { logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, diff --git a/bundle/config/validate/no_variable_reference_in_resource_key.go b/bundle/config/validate/no_variable_reference_in_resource_key.go index 5ad5b58f200..94517d5fc44 100644 --- a/bundle/config/validate/no_variable_reference_in_resource_key.go +++ b/bundle/config/validate/no_variable_reference_in_resource_key.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/logdiag" ) type noVariableReferenceInResourceKey struct{} @@ -22,7 +23,7 @@ func (m *noVariableReferenceInResourceKey) Name() string { return "validate:no_variable_reference_in_resource_key" } -func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *noVariableReferenceInResourceKey) Apply(ctx context.Context, b *bundle.Bundle) error { var diags diag.Diagnostics patterns := []dyn.Pattern{ @@ -37,7 +38,7 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu func(p dyn.Path, v dyn.Value) (dyn.Value, error) { key := p[len(p)-1].Key() if dynvar.ContainsVariableReference(key) { - diags = append(diags, diag.Diagnostic{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("resource key %q must not contain variable references", key), Locations: v.Locations(), @@ -48,9 +49,9 @@ func (m *noVariableReferenceInResourceKey) Apply(_ context.Context, b *bundle.Bu }, ) if err != nil { - diags = append(diags, diag.FromErr(err)...) + return err } } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/validate/required.go b/bundle/config/validate/required.go index 6f886caab44..991c39d484a 100644 --- a/bundle/config/validate/required.go +++ b/bundle/config/validate/required.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle/internal/validation/generated" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type required struct{} @@ -23,7 +24,7 @@ func (f *required) Name() string { } // Warn for missing fields, based on annotations in the Go SDK / OpenAPI spec. -func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func warnForMissingFields(ctx context.Context, b *bundle.Bundle) error { diags := diag.Diagnostics{} // Generate prefix tree for all required fields. @@ -31,12 +32,12 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic for k := range generated.RequiredFields { pattern, err := dyn.NewPatternFromString(k) if err != nil { - return diag.FromErr(fmt.Errorf("invalid pattern %q for required field validation: %w", k, err)) + return fmt.Errorf("invalid pattern %q for required field validation: %w", k, err) } err = trie.Insert(pattern) if err != nil { - return diag.FromErr(fmt.Errorf("failed to insert pattern %q into trie: %w", k, err)) + return fmt.Errorf("failed to insert pattern %q into trie: %w", k, err) } } @@ -65,7 +66,7 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic return nil }) if err != nil { - return diag.FromErr(err) + return err } // Sort diagnostics to make them deterministic @@ -79,18 +80,21 @@ func warnForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostic return cmp.Compare(fmt.Sprintf("%v", a.Locations), fmt.Sprintf("%v", b.Locations)) }) - return diags + for _, d := range diags { + logdiag.LogDiag(ctx, d) + } + + return nil } // Bespoke code to error for fields that are not marked as required in the Go SDK / OpenAPI spec. -func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func errorForMissingFields(ctx context.Context, b *bundle.Bundle) error { // Dashboards should always have a name and warehouse_id. var nameLocations []dyn.Location var namePaths []dyn.Path var warehouseIdLocations []dyn.Location var warehouseIdPaths []dyn.Path - diags := diag.Diagnostics{} for key, dashboard := range b.Config.Resources.Dashboards { if dashboard.DisplayName == "" { nameLocations = append(nameLocations, b.Config.GetLocations("resources.dashboards."+key)...) @@ -102,6 +106,7 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti } } + var diags diag.Diagnostics if len(nameLocations) > 0 { diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, @@ -119,14 +124,12 @@ func errorForMissingFields(ctx context.Context, b *bundle.Bundle) diag.Diagnosti }) } - return diags + return logdiag.Flush(ctx, diags) } -func (f *required) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := errorForMissingFields(ctx, b) - if diags.HasError() { - return diags +func (f *required) Apply(ctx context.Context, b *bundle.Bundle) error { + if err := errorForMissingFields(ctx, b); err != nil { + return err } - diags = diags.Extend(warnForMissingFields(ctx, b)) - return diags + return warnForMissingFields(ctx, b) } diff --git a/bundle/config/validate/scripts.go b/bundle/config/validate/scripts.go index 04c6045bb42..71fe59a5f4c 100644 --- a/bundle/config/validate/scripts.go +++ b/bundle/config/validate/scripts.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type validateScripts struct{} @@ -22,11 +23,11 @@ func (f *validateScripts) Name() string { return "validate:scripts" } -func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := diag.Diagnostics{} - +func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) error { re := regexp.MustCompile(`\$\{.*\}`) + var diags diag.Diagnostics + // Sort the scripts to have a deterministic order for the // generated diagnostics. scriptKeys := slices.Sorted(maps.Keys(b.Config.Scripts)) @@ -36,7 +37,7 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag p := dyn.NewPath(dyn.Key("scripts"), dyn.Key(k), dyn.Key("content")) if script.Content == "" { - diags = append(diags, diag.Diagnostic{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("Script %s has no content", k), Paths: []dyn.Path{p}, @@ -46,13 +47,13 @@ func (f *validateScripts) Apply(ctx context.Context, b *bundle.Bundle) diag.Diag v, err := dyn.GetByPath(b.Config.Value(), p) if err != nil { - return diags.Extend(diag.FromErr(err)) + return err } // Check for interpolation syntax match := re.FindString(script.Content) if match != "" { - diags = append(diags, diag.Diagnostic{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("Found %s in script %s. Interpolation syntax ${...} is not allowed in scripts", match, k), Detail: `We do not support the ${...} interpolation syntax in scripts because @@ -64,5 +65,5 @@ environment variable.`, } } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/validate/single_node_cluster.go b/bundle/config/validate/single_node_cluster.go index f4f6f0e5954..4fdd9c4310f 100644 --- a/bundle/config/validate/single_node_cluster.go +++ b/bundle/config/validate/single_node_cluster.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" ) // Validates that any single node clusters defined in the bundle are correctly configured. @@ -105,9 +106,7 @@ func showSingleNodeClusterWarning(ctx context.Context, v dyn.Value) bool { return false } -func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := diag.Diagnostics{} - +func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) error { patterns := []dyn.Pattern{ // Interactive clusters dyn.NewPattern(dyn.Key("resources"), dyn.Key("clusters"), dyn.AnyKey()), @@ -132,7 +131,7 @@ func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Di } if showSingleNodeClusterWarning(ctx, v) { - diags = append(diags, warning) + logdiag.LogDiag(ctx, warning) } return v, nil }) @@ -140,5 +139,5 @@ func (m *singleNodeCluster) Apply(ctx context.Context, b *bundle.Bundle) diag.Di log.Debugf(ctx, "Error while applying single node cluster validation: %s", err) } } - return diags + return nil } diff --git a/bundle/config/validate/single_node_cluster_test.go b/bundle/config/validate/single_node_cluster_test.go index bbda86f2ff3..92ee9db831e 100644 --- a/bundle/config/validate/single_node_cluster_test.go +++ b/bundle/config/validate/single_node_cluster_test.go @@ -115,7 +115,7 @@ func TestValidateSingleNodeClusterFailForInteractiveClusters(t *testing.T) { bundletest.Mutate(t, b, func(v dyn.Value) (dyn.Value, error) { return dyn.Set(v, "resources.clusters.foo.num_workers", dyn.V(0)) }) - diags := SingleNodeCluster().Apply(ctx, b) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, @@ -164,7 +164,7 @@ func TestValidateSingleNodeClusterFailForJobClusters(t *testing.T) { return dyn.Set(v, "resources.jobs.foo.job_clusters[0].new_cluster.num_workers", dyn.V(0)) }) - diags := SingleNodeCluster().Apply(ctx, b) + diags := bundle.Apply(ctx, b, SingleNodeCluster()) assert.Equal(t, diag.Diagnostics{ { Severity: diag.Warning, diff --git a/bundle/config/validate/tf_only_references.go b/bundle/config/validate/tf_only_references.go index 90de6d12468..d4af301b079 100644 --- a/bundle/config/validate/tf_only_references.go +++ b/bundle/config/validate/tf_only_references.go @@ -11,6 +11,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structpath" ) @@ -30,7 +31,7 @@ func (m *tfOnlyReferences) Name() string { return "validate:tf_only_references" } -func (m *tfOnlyReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *tfOnlyReferences) Apply(ctx context.Context, b *bundle.Bundle) error { // Resolve effective engine: config takes precedence over env var. effectiveEngine := b.Config.Bundle.Engine if effectiveEngine == engine.EngineNotSet { @@ -63,7 +64,7 @@ func (m *tfOnlyReferences) Apply(ctx context.Context, b *bundle.Bundle) diag.Dia b.Metrics.AddBoolValue("has_tf_only_references", true) } - return diags + return logdiag.Flush(ctx, diags) } // checkTFOnlyReference checks a single reference string like @@ -73,7 +74,7 @@ func checkTFOnlyReference(ref string, loc dyn.Location, isDirect bool) *diag.Dia p, err := dyn.NewPathFromString(ref) // Need at least resources... if err != nil || len(p) < 4 || p[0].Key() != "resources" { - return nil + return nil //nolint:nilerr // an unparseable reference is simply not a TF-only reference } group := p[1].Key() @@ -85,7 +86,7 @@ func checkTFOnlyReference(ref string, loc dyn.Location, isDirect bool) *diag.Dia // Field path is everything after resources... fieldNode, err := structpath.ParsePath(p[3:].String()) if err != nil { - return nil + return nil //nolint:nilerr // an unparseable field path is simply not a TF-only reference } if !tfOnlyFields.Contains(fieldNode) { diff --git a/bundle/config/validate/tf_only_references_test.go b/bundle/config/validate/tf_only_references_test.go index fdf81e3c75b..550367a82f8 100644 --- a/bundle/config/validate/tf_only_references_test.go +++ b/bundle/config/validate/tf_only_references_test.go @@ -38,7 +38,7 @@ func TestTFOnlyReferences_DirectError(t *testing.T) { }) ctx := env.Set(t.Context(), engine.EnvVar, "direct") - diags := TFOnlyReferences().Apply(ctx, b) + diags := bundle.Apply(ctx, b, TFOnlyReferences()) require.Len(t, diags, 1) assert.Equal(t, diag.Error, diags[0].Severity) assert.Contains(t, diags[0].Summary, "resources.jobs.src.always_running") @@ -52,7 +52,7 @@ func TestTFOnlyReferences_TerraformWarning(t *testing.T) { }) ctx := env.Set(t.Context(), engine.EnvVar, "terraform") - diags := TFOnlyReferences().Apply(ctx, b) + diags := bundle.Apply(ctx, b, TFOnlyReferences()) require.Len(t, diags, 1) assert.Equal(t, diag.Warning, diags[0].Severity) assert.Contains(t, diags[0].Summary, "resources.jobs.src.always_running") @@ -67,7 +67,7 @@ func TestTFOnlyReferences_NormalReference(t *testing.T) { }) ctx := env.Set(t.Context(), engine.EnvVar, "direct") - diags := TFOnlyReferences().Apply(ctx, b) + diags := bundle.Apply(ctx, b, TFOnlyReferences()) assert.Empty(t, diags) } @@ -79,6 +79,6 @@ func TestTFOnlyReferences_RenamedField(t *testing.T) { }) ctx := env.Set(t.Context(), engine.EnvVar, "direct") - diags := TFOnlyReferences().Apply(ctx, b) + diags := bundle.Apply(ctx, b, TFOnlyReferences()) assert.Empty(t, diags) } diff --git a/bundle/config/validate/unique_resource_keys.go b/bundle/config/validate/unique_resource_keys.go index 12c13fd1a86..f265e2b7649 100644 --- a/bundle/config/validate/unique_resource_keys.go +++ b/bundle/config/validate/unique_resource_keys.go @@ -3,11 +3,13 @@ package validate import ( "cmp" "context" + "maps" "slices" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) // This mutator validates that: @@ -31,8 +33,8 @@ func (m *uniqueResourceKeys) Name() string { return "validate:unique_resource_keys" } -func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := diag.Diagnostics{} +func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) error { + var diags diag.Diagnostics type metadata struct { locations []dyn.Location @@ -70,7 +72,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D }, ) if err != nil { - return diag.FromErr(err) + return err } } @@ -88,12 +90,14 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D }, ) if err != nil { - return diag.FromErr(err) + return err } } - // If duplicate keys are found, report an error. - for k, v := range resourceAndScriptMetadata { + // If duplicate keys are found, report an error. Iterate in sorted key order + // so the errors we return are deterministic. + for _, k := range slices.Sorted(maps.Keys(resourceAndScriptMetadata)) { + v := resourceAndScriptMetadata[k] if len(v.locations) <= 1 { continue } @@ -114,7 +118,7 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D }) // If there are multiple resources with the same key, report an error. - diags = append(diags, diag.Diagnostic{ + diags = diags.Append(diag.Diagnostic{ Severity: diag.Error, Summary: "multiple resources or scripts have been defined with the same key: " + k, Locations: v.locations, @@ -122,5 +126,5 @@ func (m *uniqueResourceKeys) Apply(ctx context.Context, b *bundle.Bundle) diag.D }) } - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/validate/validate.go b/bundle/config/validate/validate.go index e531d87684c..570d13b8bbf 100644 --- a/bundle/config/validate/validate.go +++ b/bundle/config/validate/validate.go @@ -6,8 +6,8 @@ import ( "github.com/databricks/cli/bundle" ) -func Validate(ctx context.Context, b *bundle.Bundle) { - bundle.ApplyParallel(ctx, b, +func Validate(ctx context.Context, b *bundle.Bundle) error { + return bundle.ApplyParallel(ctx, b, FastValidate(), // Slow mutators that require network or file i/o. These are only diff --git a/bundle/config/validate/validate_artifact_path.go b/bundle/config/validate/validate_artifact_path.go index 4ea5c4308ad..271aea2e8ab 100644 --- a/bundle/config/validate/validate_artifact_path.go +++ b/bundle/config/validate/validate_artifact_path.go @@ -74,20 +74,18 @@ func findVolumeInBundle(r config.Root, catalogName, schemaName, volumeName strin return nil, nil, false } -func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) error { // We only validate UC Volumes paths right now. if !libraries.IsVolumesPath(b.Config.Workspace.ArtifactPath) { return nil } - wrapErrorMsg := func(s string) diag.Diagnostics { - return diag.Diagnostics{ - { - Summary: s, - Severity: diag.Error, - Locations: b.Config.GetLocations("workspace.artifact_path"), - Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, - }, + wrapErrorMsg := func(s string) error { + return diag.Diagnostic{ + Summary: s, + Severity: diag.Error, + Locations: b.Config.GetLocations("workspace.artifact_path"), + Paths: []dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, } } @@ -110,7 +108,7 @@ func (v *validateArtifactPath) Apply(ctx context.Context, b *bundle.Bundle) diag // If the volume is defined in the bundle, provide a more helpful error diagnostic, // with more details and location information. - return diag.Diagnostics{{ + return diag.Diagnostic{ Summary: fmt.Sprintf("volume %s does not exist", volumeFullName), Severity: diag.Error, Detail: `You are using a volume in your artifact_path that is managed by @@ -119,7 +117,7 @@ the volume using 'bundle deploy' and then switch over to using it in the artifact_path.`, Locations: slices.Concat(b.Config.GetLocations("workspace.artifact_path"), locations), Paths: append([]dyn.Path{dyn.MustPathFromString("workspace.artifact_path")}, path), - }} + } } if err != nil { diff --git a/bundle/config/validate/validate_artifact_path_test.go b/bundle/config/validate/validate_artifact_path_test.go index 08f50e39fb8..9eae1dce355 100644 --- a/bundle/config/validate/validate_artifact_path_test.go +++ b/bundle/config/validate/validate_artifact_path_test.go @@ -48,7 +48,7 @@ func TestValidateArtifactPathWithVolumeInBundle(t *testing.T) { }) b.SetWorkpaceClient(m.WorkspaceClient) - diags := ValidateArtifactPath().Apply(ctx, b) + diags := bundle.Apply(ctx, b, ValidateArtifactPath()) assert.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: "volume catalogN.schemaN.volumeN does not exist", @@ -121,7 +121,7 @@ func TestValidateArtifactPath(t *testing.T) { api.EXPECT().ReadByName(mock.Anything, "catalogN.schemaN.volumeN").Return(nil, tc.err) b.SetWorkpaceClient(m.WorkspaceClient) - diags := ValidateArtifactPath().Apply(ctx, b) + diags := bundle.Apply(ctx, b, ValidateArtifactPath()) assertDiags(t, diags, tc.expectedSummary) } } @@ -165,7 +165,7 @@ func TestValidateArtifactPathWithInvalidPaths(t *testing.T) { bundletest.SetLocation(b, "workspace.artifact_path", []dyn.Location{{File: "config.yml", Line: 1, Column: 2}}) - diags := ValidateArtifactPath().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateArtifactPath()) require.Equal(t, diag.Diagnostics{{ Severity: diag.Error, Summary: "expected UC volume path to be in the format /Volumes////..., got " + p, diff --git a/bundle/config/validate/validate_dashboard_etags.go b/bundle/config/validate/validate_dashboard_etags.go index 53c428af1dd..4e35a66595e 100644 --- a/bundle/config/validate/validate_dashboard_etags.go +++ b/bundle/config/validate/validate_dashboard_etags.go @@ -19,17 +19,15 @@ func (v *validateDashboardEtags) Name() string { return "validate:validate_dashboard_etags" } -func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validateDashboardEtags) Apply(ctx context.Context, b *bundle.Bundle) error { // No dashboards should have etags set. They are purely internal state. for k, dashboard := range b.Config.Resources.Dashboards { if dashboard.Etag != "" { - return diag.Diagnostics{ - { - Severity: diag.Error, - Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), - Paths: []dyn.Path{dyn.MustPathFromString("resources.dashboards." + k)}, - Locations: b.Config.GetLocations("resources.dashboards." + k), - }, + return diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("dashboard %q has an etag set. Etags must not be set in bundle configuration", dashboard.DisplayName), + Paths: []dyn.Path{dyn.MustPathFromString("resources.dashboards." + k)}, + Locations: b.Config.GetLocations("resources.dashboards." + k), } } } diff --git a/bundle/config/validate/validate_deployment_fields.go b/bundle/config/validate/validate_deployment_fields.go index f551c54661b..fc891ded57f 100644 --- a/bundle/config/validate/validate_deployment_fields.go +++ b/bundle/config/validate/validate_deployment_fields.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) func ValidateDeploymentFields() bundle.ReadOnlyMutator { @@ -20,7 +21,7 @@ func (v *validateDeploymentFields) Name() string { return "validate:validate_deployment_fields" } -func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validateDeploymentFields) Apply(ctx context.Context, b *bundle.Bundle) error { var diags diag.Diagnostics // deployment_id and version_id identify the bundle deployment and its version @@ -57,5 +58,5 @@ func (v *validateDeploymentFields) Apply(_ context.Context, b *bundle.Bundle) di return cmp.Compare(x.Paths[0].String(), y.Paths[0].String()) }) - return diags + return logdiag.Flush(ctx, diags) } diff --git a/bundle/config/validate/validate_deployment_fields_test.go b/bundle/config/validate/validate_deployment_fields_test.go index 766242bc8f6..941d9a44fb0 100644 --- a/bundle/config/validate/validate_deployment_fields_test.go +++ b/bundle/config/validate/validate_deployment_fields_test.go @@ -53,7 +53,7 @@ func TestValidateDeploymentFieldsRejectsReservedFields(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - diags := ValidateDeploymentFields().Apply(t.Context(), tt.b) + diags := bundle.Apply(t.Context(), tt.b, ValidateDeploymentFields()) require.Len(t, diags, 1) assert.Equal(t, diag.Error, diags[0].Severity) assert.Equal(t, tt.want, diags[0].Summary) @@ -75,6 +75,6 @@ func TestValidateDeploymentFieldsReportsAllOffenders(t *testing.T) { }, } - diags := ValidateDeploymentFields().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateDeploymentFields()) require.Len(t, diags, 2) } diff --git a/bundle/config/validate/validate_engine.go b/bundle/config/validate/validate_engine.go index b688f644ba9..791af1da5d6 100644 --- a/bundle/config/validate/validate_engine.go +++ b/bundle/config/validate/validate_engine.go @@ -21,7 +21,7 @@ func (v *validateEngine) Name() string { return "validate:engine" } -func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) error { configEngine := b.Config.Bundle.Engine if configEngine == engine.EngineNotSet { return nil @@ -30,11 +30,11 @@ func (v *validateEngine) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnos if _, ok := engine.Parse(string(configEngine)); !ok { val := dyn.GetValue(b.Config.Value(), "bundle.engine") loc := val.Location() - return diag.Diagnostics{{ + return diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("invalid value %q for bundle.engine (expected %q or %q)", configEngine, engine.EngineTerraform, engine.EngineDirect), Locations: []dyn.Location{loc}, - }} + } } return nil diff --git a/bundle/config/validate/validate_engine_test.go b/bundle/config/validate/validate_engine_test.go index f260b631f5c..c2ab82f1bbf 100644 --- a/bundle/config/validate/validate_engine_test.go +++ b/bundle/config/validate/validate_engine_test.go @@ -22,7 +22,7 @@ func TestValidateEngineValid(t *testing.T) { }, } bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) - diags := ValidateEngine().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateEngine()) assert.Empty(t, diags) } } @@ -31,7 +31,7 @@ func TestValidateEngineNotSet(t *testing.T) { b := &bundle.Bundle{ Config: config.Root{}, } - diags := ValidateEngine().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateEngine()) assert.Empty(t, diags) } @@ -44,7 +44,7 @@ func TestValidateEngineInvalid(t *testing.T) { }, } bundletest.SetLocation(b, "bundle.engine", []dyn.Location{{File: "databricks.yml", Line: 5, Column: 3}}) - diags := ValidateEngine().Apply(t.Context(), b) + diags := bundle.Apply(t.Context(), b, ValidateEngine()) assert.Len(t, diags, 1) assert.Equal(t, diag.Error, diags[0].Severity) assert.Contains(t, diags[0].Summary, "invalid") diff --git a/bundle/config/validate/validate_genie_space_etags.go b/bundle/config/validate/validate_genie_space_etags.go index 84e9626c0f4..d90e5e38be4 100644 --- a/bundle/config/validate/validate_genie_space_etags.go +++ b/bundle/config/validate/validate_genie_space_etags.go @@ -19,19 +19,17 @@ func (v *validateGenieSpaceEtags) Name() string { return "validate:validate_genie_space_etags" } -func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validateGenieSpaceEtags) Apply(ctx context.Context, b *bundle.Bundle) error { // No genie spaces should have etags set. They are purely internal state // (persisted by the direct engine for drift detection), never authored by // the user. Mirrors ValidateDashboardEtags. for k, genieSpace := range b.Config.Resources.GenieSpaces { if genieSpace.Etag != "" { - return diag.Diagnostics{ - { - Severity: diag.Error, - Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), - Paths: []dyn.Path{dyn.MustPathFromString("resources.genie_spaces." + k)}, - Locations: b.Config.GetLocations("resources.genie_spaces." + k), - }, + return diag.Diagnostic{ + Severity: diag.Error, + Summary: fmt.Sprintf("genie space %q has an etag set. Etags must not be set in bundle configuration", genieSpace.Title), + Paths: []dyn.Path{dyn.MustPathFromString("resources.genie_spaces." + k)}, + Locations: b.Config.GetLocations("resources.genie_spaces." + k), } } } diff --git a/bundle/config/validate/validate_sync_patterns.go b/bundle/config/validate/validate_sync_patterns.go index 8488aefebd2..d6dfd055c32 100644 --- a/bundle/config/validate/validate_sync_patterns.go +++ b/bundle/config/validate/validate_sync_patterns.go @@ -23,19 +23,17 @@ func (v *validateSyncPatterns) Name() string { return "validate:validate_sync_patterns" } -func (v *validateSyncPatterns) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (v *validateSyncPatterns) Apply(ctx context.Context, b *bundle.Bundle) error { s := b.Config.Sync - checkPatterns(ctx, s.Exclude, "sync.exclude", b) - if logdiag.HasError(ctx) { - return nil + if err := checkPatterns(ctx, s.Exclude, "sync.exclude", b); err != nil { + return err } - checkPatterns(ctx, s.Include, "sync.include", b) - return nil + return checkPatterns(ctx, s.Include, "sync.include", b) } -func checkPatterns(ctx context.Context, patterns []string, path string, b *bundle.Bundle) { +func checkPatterns(ctx context.Context, patterns []string, path string, b *bundle.Bundle) error { var errs errgroup.Group for index, pattern := range patterns { @@ -69,8 +67,5 @@ func checkPatterns(ctx context.Context, patterns []string, path string, b *bundl }) } - err := errs.Wait() - if err != nil { - logdiag.LogError(ctx, err) - } + return errs.Wait() } diff --git a/bundle/config/validate/validate_volume_path.go b/bundle/config/validate/validate_volume_path.go index 265a0bfd53b..9714d10aead 100644 --- a/bundle/config/validate/validate_volume_path.go +++ b/bundle/config/validate/validate_volume_path.go @@ -18,7 +18,7 @@ func (m *validateVolumePath) Name() string { return "validate:volume-path" } -func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) error { var diags diag.Diagnostics // Define paths to check and their corresponding config field names pathChecks := []struct { @@ -44,12 +44,12 @@ func (m *validateVolumePath) Apply(ctx context.Context, b *bundle.Bundle) diag.D // Return early for root path validation if check.configName == "workspace.root_path" { - return diags + return diags.Error() } } } - return diags + return diags.Error() } func ValidateVolumePath() bundle.ReadOnlyMutator { diff --git a/bundle/configsync/patch_test.go b/bundle/configsync/patch_test.go index b4faea82858..2f25c54fa03 100644 --- a/bundle/configsync/patch_test.go +++ b/bundle/configsync/patch_test.go @@ -39,7 +39,7 @@ resources: b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) changes := Changes{ "resources.jobs.test_job": ResourceChanges{ diff --git a/bundle/configsync/resolve_test.go b/bundle/configsync/resolve_test.go index 9264ad7f5dc..21e25387079 100644 --- a/bundle/configsync/resolve_test.go +++ b/bundle/configsync/resolve_test.go @@ -30,7 +30,7 @@ func TestResolveSelectors_NoSelectors(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) result, _, err := resolveSelectors("resources.jobs.test_job.name", b, OperationReplace) require.NoError(t, err) @@ -55,7 +55,7 @@ func TestResolveSelectors_NumericIndices(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) result, _, err := resolveSelectors("resources.jobs.test_job.tasks[0].task_key", b, OperationReplace) require.NoError(t, err) @@ -88,7 +88,7 @@ func TestResolveSelectors_KeyValueSelector(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) result, _, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].notebook_task.notebook_path", b, OperationReplace) require.NoError(t, err) @@ -118,7 +118,7 @@ func TestResolveSelectors_SelectorNotFound(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) _, _, err = resolveSelectors("resources.jobs.test_job.tasks[task_key='nonexistent'].notebook_task.notebook_path", b, OperationReplace) require.Error(t, err) @@ -130,9 +130,9 @@ func TestResolveSelectors_SelectorOnNonArray(t *testing.T) { tmpDir := t.TempDir() yamlContent := `resources: - jobs: - test_job: - name: "Test Job" + jobs: + test_job: + name: "Test Job" ` yamlPath := filepath.Join(tmpDir, "databricks.yml") err := os.WriteFile(yamlPath, []byte(yamlContent), 0o644) @@ -141,7 +141,7 @@ func TestResolveSelectors_SelectorOnNonArray(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) _, _, err = resolveSelectors("resources.jobs.test_job[task_key='main'].name", b, OperationReplace) require.Error(t, err) @@ -172,7 +172,7 @@ func TestResolveSelectors_NestedSelectors(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) result, _, err := resolveSelectors("resources.jobs.test_job.tasks[task_key='main'].libraries[0].pypi.package", b, OperationReplace) require.NoError(t, err) @@ -198,7 +198,7 @@ func TestResolveSelectors_WildcardNotSupported(t *testing.T) { b, err := bundle.Load(ctx, tmpDir) require.NoError(t, err) - mutator.DefaultMutators(ctx, b) + require.NoError(t, mutator.DefaultMutators(ctx, b)) _, _, err = resolveSelectors("resources.jobs.test_job.tasks.*.task_key", b, OperationReplace) require.Error(t, err) diff --git a/bundle/configsync/variables.go b/bundle/configsync/variables.go index 055a47dc934..303fb5b2bb6 100644 --- a/bundle/configsync/variables.go +++ b/bundle/configsync/variables.go @@ -127,10 +127,14 @@ func loadPreResolvedConfig(ctx context.Context, b *bundle.Bundle) dyn.Value { BundleRootPath: b.BundleRootPath, BundleRoot: b.BundleRoot, } - mutator.DefaultMutators(ctx, fresh) + if err := mutator.DefaultMutators(ctx, fresh); err != nil { + return dyn.InvalidValue + } if target := b.Config.Bundle.Target; target != "" { if _, ok := fresh.Config.Targets[target]; ok { - bundle.ApplyContext(ctx, fresh, mutator.SelectTarget(target)) + if err := bundle.ApplyContext(ctx, fresh, mutator.SelectTarget(target)); err != nil { + return dyn.InvalidValue + } } } return fresh.Config.Value() diff --git a/bundle/deploy/files/delete.go b/bundle/deploy/files/delete.go index aab6cca9acc..d53c1861390 100644 --- a/bundle/deploy/files/delete.go +++ b/bundle/deploy/files/delete.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/sync" "github.com/databricks/databricks-sdk-go/service/workspace" ) @@ -20,7 +19,7 @@ func (m *delete) Name() string { return "files.Delete" } -func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) error { cmdio.LogString(ctx, "Deleting files...") err := b.WorkspaceClient(ctx).Workspace.Delete(ctx, workspace.Delete{ @@ -28,13 +27,13 @@ func (m *delete) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { Recursive: true, }) if err != nil { - return diag.FromErr(err) + return err } // Clean up sync snapshot file err = deleteSnapshotFile(ctx, b) if err != nil { - return diag.FromErr(err) + return err } return nil } diff --git a/bundle/deploy/files/upload.go b/bundle/deploy/files/upload.go index bb46c97c999..7e0f1228297 100644 --- a/bundle/deploy/files/upload.go +++ b/bundle/deploy/files/upload.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/permissions" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/sync" ) @@ -23,7 +22,7 @@ func (m *upload) Name() string { return "files.Upload" } -func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { cmdio.LogString(ctx, "Source-linked deployment is enabled. Deployed resources reference the source files in your working tree instead of separate copies.") return nil @@ -32,13 +31,13 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { cmdio.LogString(ctx, fmt.Sprintf("Uploading bundle files to %s...", b.Config.Workspace.FilePath)) opts, err := GetSyncOptions(ctx, b) if err != nil { - return diag.FromErr(err) + return err } opts.OutputHandler = m.outputHandler sync, err := sync.New(ctx, *opts) if err != nil { - return diag.FromErr(err) + return err } defer sync.Close() @@ -47,7 +46,7 @@ func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { if errors.Is(err, fs.ErrPermission) { return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.FilePath) } - return diag.FromErr(err) + return err } log.Infof(ctx, "Uploaded bundle files") diff --git a/bundle/deploy/lock/acquire.go b/bundle/deploy/lock/acquire.go index 6e4844ca5ff..edf6ded4859 100644 --- a/bundle/deploy/lock/acquire.go +++ b/bundle/deploy/lock/acquire.go @@ -7,7 +7,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/locker" "github.com/databricks/cli/libs/log" ) @@ -34,7 +33,7 @@ func (m *acquire) init(ctx context.Context, b *bundle.Bundle) error { return nil } -func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) error { // Return early if locking is disabled. if !b.Config.Bundle.Deployment.Lock.IsEnabled() { log.Infof(ctx, "Skipping; locking is disabled") @@ -43,7 +42,7 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics err := m.init(ctx, b) if err != nil { - return diag.FromErr(err) + return err } force := b.Config.Bundle.Deployment.Lock.Force @@ -62,7 +61,7 @@ func (m *acquire) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics return permissions.ReportPossiblePermissionDenied(ctx, b, b.Config.Workspace.StatePath) } - return diag.FromErr(err) + return err } return nil diff --git a/bundle/deploy/lock/release.go b/bundle/deploy/lock/release.go index 26f95edfc95..938feae4de1 100644 --- a/bundle/deploy/lock/release.go +++ b/bundle/deploy/lock/release.go @@ -30,7 +30,7 @@ func (m *release) Name() string { return "lock:release" } -func (m *release) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *release) Apply(ctx context.Context, b *bundle.Bundle) error { // Return early if locking is disabled. if !b.Config.Bundle.Deployment.Lock.IsEnabled() { log.Infof(ctx, "Skipping; locking is disabled") @@ -47,11 +47,11 @@ func (m *release) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics log.Infof(ctx, "Releasing deployment lock") switch m.goal { case GoalDeploy: - return diag.FromErr(b.Locker.Unlock(ctx)) + return b.Locker.Unlock(ctx) case GoalBind, GoalUnbind: - return diag.FromErr(b.Locker.Unlock(ctx)) + return b.Locker.Unlock(ctx) case GoalDestroy: - return diag.FromErr(b.Locker.Unlock(ctx, locker.AllowLockFileNotExist)) + return b.Locker.Unlock(ctx, locker.AllowLockFileNotExist) default: return diag.Errorf("unknown goal for lock release: %s", m.goal) } diff --git a/bundle/deploy/metadata/annotate_jobs.go b/bundle/deploy/metadata/annotate_jobs.go index 1537f6ae8f1..5f382462b07 100644 --- a/bundle/deploy/metadata/annotate_jobs.go +++ b/bundle/deploy/metadata/annotate_jobs.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/dbr" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -22,7 +21,7 @@ func (m *annotateJobs) Name() string { return "metadata.AnnotateJobs" } -func (m *annotateJobs) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *annotateJobs) Apply(ctx context.Context, b *bundle.Bundle) error { for _, job := range b.Config.Resources.Jobs { job.Deployment = &jobs.JobDeployment{ Kind: jobs.JobDeploymentKindBundle, diff --git a/bundle/deploy/metadata/annotate_jobs_test.go b/bundle/deploy/metadata/annotate_jobs_test.go index 767559b19db..837046582c4 100644 --- a/bundle/deploy/metadata/annotate_jobs_test.go +++ b/bundle/deploy/metadata/annotate_jobs_test.go @@ -40,7 +40,7 @@ func TestAnnotateJobsMutator(t *testing.T) { ctx = dbr.MockRuntime(ctx, dbr.Environment{IsDbr: false, Version: ""}) diags := AnnotateJobs().Apply(ctx, b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, &jobs.JobDeployment{ @@ -76,7 +76,7 @@ func TestAnnotateJobsMutatorJobWithoutSettings(t *testing.T) { ctx = dbr.MockRuntime(ctx, dbr.Environment{IsDbr: false, Version: ""}) diags := AnnotateJobs().Apply(ctx, b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) } func TestAnnotateJobsWorkspaceWithFlag(t *testing.T) { @@ -106,7 +106,7 @@ func TestAnnotateJobsWorkspaceWithFlag(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC", "1") diags := AnnotateJobs().Apply(ctx, b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, jobs.JobEditModeEditable, b.Config.Resources.Jobs["my-job"].EditMode) assert.Equal(t, jobs.FormatMultiTask, b.Config.Resources.Jobs["my-job"].Format) @@ -139,7 +139,7 @@ func TestAnnotateJobsWorkspaceNonDevelopment(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC", "1") diags := AnnotateJobs().Apply(ctx, b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, jobs.JobEditModeUiLocked, b.Config.Resources.Jobs["my-job"].EditMode) assert.Equal(t, jobs.FormatMultiTask, b.Config.Resources.Jobs["my-job"].Format) @@ -168,7 +168,7 @@ func TestAnnotateJobsWorkspaceWithoutFlag(t *testing.T) { ctx = dbr.MockRuntime(ctx, dbr.Environment{IsDbr: true, Version: "14.0"}) diags := AnnotateJobs().Apply(ctx, b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, jobs.JobEditModeUiLocked, b.Config.Resources.Jobs["my-job"].EditMode) assert.Equal(t, jobs.FormatMultiTask, b.Config.Resources.Jobs["my-job"].Format) @@ -198,7 +198,7 @@ func TestAnnotateJobsNonWorkspaceWithFlag(t *testing.T) { ctx = env.Set(ctx, "DATABRICKS_BUNDLE_ENABLE_EXPERIMENTAL_YAML_SYNC", "1") diags := AnnotateJobs().Apply(ctx, b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, jobs.JobEditModeUiLocked, b.Config.Resources.Jobs["my-job"].EditMode) assert.Equal(t, jobs.FormatMultiTask, b.Config.Resources.Jobs["my-job"].Format) diff --git a/bundle/deploy/metadata/annotate_pipelines.go b/bundle/deploy/metadata/annotate_pipelines.go index 284766d8971..5d28886789d 100644 --- a/bundle/deploy/metadata/annotate_pipelines.go +++ b/bundle/deploy/metadata/annotate_pipelines.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/pipelines" ) @@ -18,7 +17,7 @@ func (m *annotatePipelines) Name() string { return "metadata.AnnotatePipelines" } -func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *annotatePipelines) Apply(_ context.Context, b *bundle.Bundle) error { for _, pipeline := range b.Config.Resources.Pipelines { pipeline.Deployment = &pipelines.PipelineDeployment{ Kind: pipelines.DeploymentKindBundle, diff --git a/bundle/deploy/metadata/compute.go b/bundle/deploy/metadata/compute.go index cb7be9811c4..449181ef711 100644 --- a/bundle/deploy/metadata/compute.go +++ b/bundle/deploy/metadata/compute.go @@ -22,7 +22,7 @@ func (m *compute) Name() string { return "metadata.Compute" } -func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *compute) Apply(ctx context.Context, b *bundle.Bundle) error { b.Metadata = metadata.Metadata{ Version: metadata.Version, Config: metadata.Config{}, diff --git a/bundle/deploy/metadata/upload.go b/bundle/deploy/metadata/upload.go index 79546518a36..e348f0794a7 100644 --- a/bundle/deploy/metadata/upload.go +++ b/bundle/deploy/metadata/upload.go @@ -7,7 +7,6 @@ import ( "path" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" ) @@ -27,16 +26,16 @@ func (m *upload) Name() string { return "metadata.Upload" } -func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *upload) Apply(ctx context.Context, b *bundle.Bundle) error { f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(ctx), b.Config.Workspace.StatePath) if err != nil { - return diag.FromErr(err) + return err } metadata, err := json.MarshalIndent(b.Metadata, "", " ") if err != nil { - return diag.FromErr(err) + return err } - return diag.FromErr(f.Write(ctx, metadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists)) + return f.Write(ctx, metadataFileName, bytes.NewReader(metadata), filer.CreateParentDirectories, filer.OverwriteIfExists) } diff --git a/bundle/deploy/resource_path_mkdir.go b/bundle/deploy/resource_path_mkdir.go index 5c41a270a80..45a1df77cf5 100644 --- a/bundle/deploy/resource_path_mkdir.go +++ b/bundle/deploy/resource_path_mkdir.go @@ -5,7 +5,6 @@ import ( "errors" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/apierr" ) @@ -19,7 +18,7 @@ func (m *resourcePathMkdir) Name() string { return "deploy:resource_path_mkdir" } -func (m *resourcePathMkdir) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *resourcePathMkdir) Apply(ctx context.Context, b *bundle.Bundle) error { // Only dashboards and alerts need ${workspace.resource_path} to exist. if len(b.Config.Resources.Alerts) == 0 && len(b.Config.Resources.Dashboards) == 0 { return nil @@ -32,5 +31,5 @@ func (m *resourcePathMkdir) Apply(ctx context.Context, b *bundle.Bundle) diag.Di if aerr, ok := errors.AsType[*apierr.APIError](err); ok && aerr.ErrorCode == "RESOURCE_ALREADY_EXISTS" { return nil } - return diag.FromErr(err) + return err } diff --git a/bundle/deploy/state_pull.go b/bundle/deploy/state_pull.go index 832fac87fbb..f65cc5d0363 100644 --- a/bundle/deploy/state_pull.go +++ b/bundle/deploy/state_pull.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/deploy/files" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/sync" @@ -21,10 +20,10 @@ type statePull struct { filerFactory FilerFactory } -func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) error { f, err := s.filerFactory(ctx, b) if err != nil { - return diag.FromErr(err) + return err } // Download deployment state file from filer to local cache directory. @@ -32,7 +31,7 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic remote, err := s.remoteState(ctx, f) if err != nil { log.Infof(ctx, "Unable to open remote deployment state file: %s", err) - return diag.FromErr(err) + return err } if remote == nil { log.Infof(ctx, "Remote deployment state file does not exist") @@ -41,19 +40,19 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic statePath, err := getPathToStateFile(ctx, b) if err != nil { - return diag.FromErr(err) + return err } local, err := os.OpenFile(statePath, os.O_CREATE|os.O_RDWR, 0o600) if err != nil { - return diag.FromErr(err) + return err } defer local.Close() data := remote.Bytes() err = validateRemoteStateCompatibility(bytes.NewReader(data)) if err != nil { - return diag.FromErr(err) + return err } if !isLocalStateStale(local, bytes.NewReader(data)) { @@ -64,41 +63,41 @@ func (s *statePull) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostic // Truncating the file before writing err = local.Truncate(0) if err != nil { - return diag.FromErr(err) + return err } _, err = local.Seek(0, 0) if err != nil { - return diag.FromErr(err) + return err } // Write file to disk. log.Infof(ctx, "Writing remote deployment state file to local cache directory") _, err = io.Copy(local, bytes.NewReader(data)) if err != nil { - return diag.FromErr(err) + return err } var state DeploymentState err = json.Unmarshal(data, &state) if err != nil { - return diag.FromErr(err) + return err } // Create a new snapshot based on the deployment state file. opts, err := files.GetSyncOptions(ctx, b) if err != nil { - return diag.FromErr(err) + return err } log.Infof(ctx, "Creating new snapshot") snapshot, err := sync.NewSnapshot(state.Files.toSlice(b.SyncRoot), opts) if err != nil { - return diag.FromErr(err) + return err } // Persist the snapshot to disk. log.Infof(ctx, "Persisting snapshot to disk") - return diag.FromErr(snapshot.Save(ctx)) + return snapshot.Save(ctx) } func (s *statePull) remoteState(ctx context.Context, f filer.Filer) (*bytes.Buffer, error) { diff --git a/bundle/deploy/state_push.go b/bundle/deploy/state_push.go index 65a886a86fe..8767a72ec51 100644 --- a/bundle/deploy/state_push.go +++ b/bundle/deploy/state_push.go @@ -5,7 +5,6 @@ import ( "os" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" ) @@ -18,27 +17,27 @@ func (s *statePush) Name() string { return "deploy:state-push" } -func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (s *statePush) Apply(ctx context.Context, b *bundle.Bundle) error { f, err := s.filerFactory(ctx, b) if err != nil { - return diag.FromErr(err) + return err } statePath, err := getPathToStateFile(ctx, b) if err != nil { - return diag.FromErr(err) + return err } local, err := os.Open(statePath) if err != nil { - return diag.FromErr(err) + return err } defer local.Close() log.Infof(ctx, "Writing local deployment state file to remote state directory") err = f.Write(ctx, DeploymentStateFileName, local, filer.CreateParentDirectories, filer.OverwriteIfExists) if err != nil { - return diag.FromErr(err) + return err } return nil diff --git a/bundle/deploy/state_update.go b/bundle/deploy/state_update.go index 55cf2393bf1..76a7f89ce14 100644 --- a/bundle/deploy/state_update.go +++ b/bundle/deploy/state_update.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/google/uuid" ) @@ -23,10 +22,10 @@ func (s *stateUpdate) Name() string { return "deploy:state-update" } -func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) error { state, err := load(ctx, b) if err != nil { - return diag.FromErr(err) + return err } // Increment the state sequence. @@ -42,7 +41,7 @@ func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost // Update the state with the current list of synced files. fl, err := fromSlice(b.Files) if err != nil { - return diag.FromErr(err) + return err } state.Files = fl @@ -54,24 +53,24 @@ func (s *stateUpdate) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnost statePath, err := getPathToStateFile(ctx, b) if err != nil { - return diag.FromErr(err) + return err } // Write the state back to the file. f, err := os.OpenFile(statePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o600) if err != nil { log.Infof(ctx, "Unable to open deployment state file: %s", err) - return diag.FromErr(err) + return err } defer f.Close() data, err := json.Marshal(state) if err != nil { - return diag.FromErr(err) + return err } _, err = io.Copy(f, bytes.NewReader(data)) if err != nil { - return diag.FromErr(err) + return err } return nil diff --git a/bundle/deploy/terraform/apply.go b/bundle/deploy/terraform/apply.go index aff2025e018..8e2f49823cd 100644 --- a/bundle/deploy/terraform/apply.go +++ b/bundle/deploy/terraform/apply.go @@ -16,7 +16,7 @@ func (w *apply) Name() string { return "terraform.Apply" } -func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) error { // return early if plan is empty if b.TerraformPlanIsEmpty { log.Debugf(ctx, "No changes in plan. Skipping terraform apply.") @@ -35,9 +35,8 @@ func (w *apply) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { // Apply terraform according to the computed plan err := tf.Apply(ctx, tfexec.DirOrPlan(b.TerraformPlanPath)) if err != nil { - diags := permissions.TryExtendTerraformPermissionError(ctx, b, err) - if diags != nil { - return diags + if extErr := permissions.TryExtendTerraformPermissionError(ctx, b, err); extErr != nil { + return extErr } return diag.Errorf("terraform apply: %v", err) } diff --git a/bundle/deploy/terraform/check_dashboards_modified_remotely.go b/bundle/deploy/terraform/check_dashboards_modified_remotely.go index ed873618ed1..9403527a81f 100644 --- a/bundle/deploy/terraform/check_dashboards_modified_remotely.go +++ b/bundle/deploy/terraform/check_dashboards_modified_remotely.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/agent" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type dashboardState struct { @@ -62,7 +63,7 @@ func (l *checkDashboardsModifiedRemotely) Name() string { return "CheckDashboardsModifiedRemotely" } -func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.Bundle) error { // This mutator is relevant only if the bundle includes dashboards. if len(b.Config.Resources.Dashboards) == 0 { return nil @@ -75,7 +76,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B dashboards, err := collectDashboardsFromState(ctx, b, l.engine.IsDirect()) if err != nil { - return diag.FromErr(err) + return err } var diags diag.Diagnostics @@ -128,7 +129,7 @@ func (l *checkDashboardsModifiedRemotely) Apply(ctx context.Context, b *bundle.B }) } - return diags + return logdiag.Flush(ctx, diags) } func CheckDashboardsModifiedRemotely(isPlan bool, engine engine.EngineType) *checkDashboardsModifiedRemotely { diff --git a/bundle/deploy/terraform/import.go b/bundle/deploy/terraform/import.go index 48ad622c6cb..5af4de6f683 100644 --- a/bundle/deploy/terraform/import.go +++ b/bundle/deploy/terraform/import.go @@ -28,15 +28,14 @@ type importResource struct { } // Apply implements bundle.Mutator. -func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) error { dir, err := Dir(ctx, b) if err != nil { - return diag.FromErr(err) + return err } - diags := Initialize(ctx, b) - if diags.HasError() { - return diags + if err := Initialize(ctx, b); err != nil { + return err } tf := b.Terraform @@ -78,7 +77,7 @@ func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") if err != nil { - return diag.FromErr(err) + return err } if !ans { return diag.Errorf("import aborted") @@ -88,22 +87,22 @@ func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagn // If user confirmed changes, move the state file from temp dir to state location f, err := os.Create(filepath.Join(dir, relPath)) if err != nil { - return diag.FromErr(err) + return err } defer f.Close() tmpF, err := os.Open(tmpState) if err != nil { - return diag.FromErr(err) + return err } defer tmpF.Close() _, err = io.Copy(f, tmpF) if err != nil { - return diag.FromErr(err) + return err } - return diags + return nil } // Name implements bundle.Mutator. diff --git a/bundle/deploy/terraform/init.go b/bundle/deploy/terraform/init.go index cbabeb3fee1..b2fc43fd0c4 100644 --- a/bundle/deploy/terraform/init.go +++ b/bundle/deploy/terraform/init.go @@ -319,7 +319,7 @@ func getTerraformExec(ctx context.Context, b *bundle.Bundle, execPath string) (* return tfexec.NewTerraform(workingDir, execPath) } -func Initialize(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func Initialize(ctx context.Context, b *bundle.Bundle) error { tfConfig := b.Config.Bundle.Terraform if tfConfig == nil { tfConfig = &config.Terraform{} @@ -328,46 +328,46 @@ func Initialize(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { execPath, err := findExecPath(ctx, b, tfConfig, tfInstaller{}) if err != nil { - return diag.FromErr(err) + return err } tfe, err := getTerraformExec(ctx, b, execPath) if err != nil { - return diag.FromErr(err) + return err } environ, err := b.AuthEnv(ctx) if err != nil { - return diag.FromErr(err) + return err } err = inheritEnvVars(ctx, environ) if err != nil { - return diag.FromErr(err) + return err } // Set the temporary directory environment variables err = setTempDirEnvVars(ctx, environ, b) if err != nil { - return diag.FromErr(err) + return err } // Set the proxy related environment variables err = setProxyEnvVars(ctx, environ, b) if err != nil { - return diag.FromErr(err) + return err } err = setUserAgentExtraEnvVar(environ, b) if err != nil { - return diag.FromErr(err) + return err } // Configure environment variables for auth for Terraform to use. log.Debugf(ctx, "Environment variables for Terraform: %s", strings.Join(slices.Collect(maps.Keys(environ)), ", ")) err = tfe.SetEnv(environ) if err != nil { - return diag.FromErr(err) + return err } err = tfe.Init(ctx, tfexec.Upgrade(true)) diff --git a/bundle/deploy/terraform/interpolate.go b/bundle/deploy/terraform/interpolate.go index edf46782743..e0ab3e784d2 100644 --- a/bundle/deploy/terraform/interpolate.go +++ b/bundle/deploy/terraform/interpolate.go @@ -5,7 +5,6 @@ import ( "fmt" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" ) @@ -31,7 +30,7 @@ func (m *interpolateMutator) Name() string { return "terraform.Interpolate" } -func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) error { err := b.Config.Mutate(func(root dyn.Value) (dyn.Value, error) { prefix := dyn.MustPathFromString("resources") @@ -74,5 +73,5 @@ func (m *interpolateMutator) Apply(ctx context.Context, b *bundle.Bundle) diag.D }) }) - return diag.FromErr(err) + return err } diff --git a/bundle/deploy/terraform/plan.go b/bundle/deploy/terraform/plan.go index 660cd0ba34b..78b2221df14 100644 --- a/bundle/deploy/terraform/plan.go +++ b/bundle/deploy/terraform/plan.go @@ -5,7 +5,6 @@ import ( "path/filepath" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" "github.com/hashicorp/terraform-exec/tfexec" ) @@ -25,23 +24,22 @@ func (p *plan) Name() string { return "terraform.Plan" } -func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := Initialize(ctx, b) - if diags.HasError() { - return diags +func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) error { + if err := Initialize(ctx, b); err != nil { + return err } // Persist computed plan tfDir, err := Dir(ctx, b) if err != nil { - return diag.FromErr(err) + return err } planPath := filepath.Join(tfDir, "plan") destroy := p.goal == PlanDestroy notEmpty, err := b.Terraform.Plan(ctx, tfexec.Destroy(destroy), tfexec.Out(planPath)) if err != nil { - return diag.FromErr(err) + return err } // Set plan in main bundle struct for downstream mutators @@ -49,7 +47,7 @@ func (p *plan) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { b.TerraformPlanIsEmpty = !notEmpty log.Debugf(ctx, "Planning complete and persisted at %s\n", planPath) - return diags + return nil } // Plan returns a [bundle.Mutator] that runs the equivalent of `terraform plan -out ./plan` diff --git a/bundle/deploy/terraform/unbind.go b/bundle/deploy/terraform/unbind.go index 25cc8271921..15024d3c4ff 100644 --- a/bundle/deploy/terraform/unbind.go +++ b/bundle/deploy/terraform/unbind.go @@ -14,10 +14,9 @@ type unbind struct { resourceKey string } -func (m *unbind) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - diags := Initialize(ctx, b) - if diags.HasError() { - return diags +func (m *unbind) Apply(ctx context.Context, b *bundle.Bundle) error { + if err := Initialize(ctx, b); err != nil { + return err } tf := b.Terraform diff --git a/bundle/deploy/terraform/write.go b/bundle/deploy/terraform/write.go index bee777ffe00..e688f6a61ce 100644 --- a/bundle/deploy/terraform/write.go +++ b/bundle/deploy/terraform/write.go @@ -8,7 +8,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/internal/tf/schema" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -18,10 +17,10 @@ func (w *write) Name() string { return "terraform.Write" } -func (w *write) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (w *write) Apply(ctx context.Context, b *bundle.Bundle) error { dir, err := Dir(ctx, b) if err != nil { - return diag.FromErr(err) + return err } var root *schema.Root @@ -30,12 +29,12 @@ func (w *write) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return v, err }) if err != nil { - return diag.FromErr(err) + return err } f, err := os.Create(filepath.Join(dir, TerraformConfigFileName)) if err != nil { - return diag.FromErr(err) + return err } defer f.Close() @@ -44,7 +43,7 @@ func (w *write) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { enc.SetIndent("", " ") err = enc.Encode(root) if err != nil { - return diag.FromErr(err) + return err } return nil diff --git a/bundle/direct/bundle_apply.go b/bundle/direct/bundle_apply.go index a63d70aee13..8b5c7c4adad 100644 --- a/bundle/direct/bundle_apply.go +++ b/bundle/direct/bundle_apply.go @@ -9,40 +9,39 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/terraform_dabs_map" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" "github.com/databricks/databricks-sdk-go" ) -func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) { +func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.WorkspaceClient, plan *deployplan.Plan) error { if plan == nil { panic("Planning is not done") } if len(plan.Plan) == 0 { // Avoid creating state file if nothing to deploy - return + return nil } b.StateDB.AssertOpenedForWrite() b.RemoteStateCache.Clear() + b.deployErrs.reset() g, err := makeGraph(plan) if err != nil { - logdiag.LogError(ctx, err) - return + return err } g.Run(defaultParallelism, func(resourceKey string, failedDependency *string) bool { entry, err := plan.WriteLockEntry(resourceKey) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: %w", resourceKey, err)) + b.deployErrs.add(fmt.Errorf("%s: internal error: %w", resourceKey, err)) return false } if entry == nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: node not in graph", resourceKey)) + b.deployErrs.add(fmt.Errorf("%s: internal error: node not in graph", resourceKey)) return false } @@ -52,21 +51,21 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa errorPrefix := fmt.Sprintf("cannot %s %s", action, resourceKey) if action == deployplan.Undefined { - logdiag.LogError(ctx, fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) + b.deployErrs.add(fmt.Errorf("cannot deploy %s: unknown action %q", resourceKey, action)) return false } // If a dependency failed, report and skip execution for this node by returning false if failedDependency != nil { if action != deployplan.Skip { - logdiag.LogError(ctx, fmt.Errorf("%s: dependency failed: %s", errorPrefix, *failedDependency)) + b.deployErrs.add(fmt.Errorf("%s: dependency failed: %s", errorPrefix, *failedDependency)) } return false } adapter, err := b.getAdapterForKey(resourceKey) if adapter == nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: internal error: cannot get adapter: %w", errorPrefix, err)) return false } @@ -79,7 +78,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if action == deployplan.Delete { err = d.Destroy(ctx, &b.StateDB) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: %w", errorPrefix, err)) return false } return true @@ -95,19 +94,19 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa // Get the cached StructVar to check for unresolved refs and get value sv, ok := b.StateCache.Load(resourceKey) if !ok { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: missing cached StructVar", errorPrefix)) + b.deployErrs.add(fmt.Errorf("%s: internal error: missing cached StructVar", errorPrefix)) return false } if len(sv.Refs) > 0 { - logdiag.LogError(ctx, fmt.Errorf("%s: unresolved references: %s", errorPrefix, jsonDump(sv.Refs))) + b.deployErrs.add(fmt.Errorf("%s: unresolved references: %s", errorPrefix, jsonDump(sv.Refs))) return false } // TODO: redo calcDiff to downgrade planned action if possible (?) err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: %w", errorPrefix, err)) return false } } @@ -119,13 +118,13 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa if needRemoteState { id := b.StateDB.GetResourceID(d.ResourceKey) if id == "" { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: missing entry in state after deploy", errorPrefix)) + b.deployErrs.add(fmt.Errorf("%s: internal error: missing entry in state after deploy", errorPrefix)) return false } err = d.refreshRemoteState(ctx, id) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: failed to read remote state: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: failed to read remote state: %w", errorPrefix, err)) return false } b.RemoteStateCache.Store(resourceKey, d.RemoteState) @@ -133,6 +132,8 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa return true }) + + return b.deployErrs.join() } func (b *DeploymentBundle) LookupReferencePostDeploy(ctx context.Context, path *structpath.PathNode) (any, error) { diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index 4f4ec5ff2d9..203ec1e9bd3 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structdiff" "github.com/databricks/cli/libs/structs/structpath" @@ -98,6 +97,7 @@ func (b *DeploymentBundle) InitForApply(ctx context.Context, client *databricks. // StateDB must already be open for read before calling this function. func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks.WorkspaceClient, configRoot *config.Root) (*deployplan.Plan, error) { b.StateDB.AssertOpenedForRead() + b.deployErrs.reset() err := b.init(client) if err != nil { @@ -128,12 +128,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks entry, err := plan.WriteLockEntry(resourceKey) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: internal error: %w", errorPrefix, err)) return false } if entry == nil { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: node not in graph", errorPrefix)) + b.deployErrs.add(fmt.Errorf("%s: internal error: node not in graph", errorPrefix)) return false } @@ -141,20 +141,20 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks if failedDependency != nil { // TODO: this should be a warning - logdiag.LogError(ctx, fmt.Errorf("%s: dependency failed: %s", errorPrefix, *failedDependency)) + b.deployErrs.add(fmt.Errorf("%s: dependency failed: %s", errorPrefix, *failedDependency)) return false } adapter, err := b.getAdapterForKey(resourceKey) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: getting adapter: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: getting adapter: %w", errorPrefix, err)) return false } if entry.Action == deployplan.Delete { id := b.StateDB.GetResourceID(resourceKey) if id == "" { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error, missing in state", errorPrefix)) + b.deployErrs.add(fmt.Errorf("%s: internal error, missing in state", errorPrefix)) return false } @@ -194,7 +194,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks savedState, err := parseState(adapter.StateType(), dbentry.State) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: interpreting state: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: interpreting state: %w", errorPrefix, err)) return false } @@ -206,12 +206,12 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // This means distinguishing between 0 that are actually object ids and 0 that are there because typed struct integer cannot contain ${...} string. sv, ok := b.StateCache.Load(resourceKey) if !ok { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no state cache entry found for %q", errorPrefix, resourceKey)) + b.deployErrs.add(fmt.Errorf("%s: internal error: no state cache entry found for %q", errorPrefix, resourceKey)) return false } localDiff, err := structdiff.GetStructDiff(savedState, sv.Value, adapter.KeyedSlices()) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: diffing local state: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: diffing local state: %w", errorPrefix, err)) return false } @@ -222,7 +222,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks if apierr.IsMissing(err) { remoteState = nil } else { - logdiag.LogError(ctx, fmt.Errorf("%s: reading id=%q: %w", errorPrefix, dbentry.ID, err)) + b.deployErrs.add(fmt.Errorf("%s: reading id=%q: %w", errorPrefix, dbentry.ID, err)) return false } } @@ -238,26 +238,26 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks if remoteState != nil { remoteStateComparable, err = adapter.RemapState(remoteState) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: interpreting remote state id=%q: %w", errorPrefix, dbentry.ID, err)) + b.deployErrs.add(fmt.Errorf("%s: interpreting remote state id=%q: %w", errorPrefix, dbentry.ID, err)) return false } remoteDiff, err = structdiff.GetStructDiff(remoteStateComparable, sv.Value, adapter.KeyedSlices()) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: diffing remote state: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: diffing remote state: %w", errorPrefix, err)) return false } } entry.Changes, err = prepareChanges(ctx, adapter, localDiff, remoteDiff, savedState, remoteStateComparable) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: %w", errorPrefix, err)) return false } err = addPerFieldActions(ctx, adapter, entry.Changes, remoteState) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: classifying changes: %w", errorPrefix, err)) return false } @@ -275,7 +275,7 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks // Validate that resources without DoUpdate don't have update actions if action == deployplan.Update && !adapter.HasDoUpdate() { - logdiag.LogError(ctx, fmt.Errorf("%s: resource does not support update action but plan produced update", errorPrefix)) + b.deployErrs.add(fmt.Errorf("%s: resource does not support update action but plan produced update", errorPrefix)) return false } @@ -283,8 +283,8 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return true }) - if logdiag.HasError(ctx) { - return nil, errors.New("planning failed") + if err := b.deployErrs.join(); err != nil { + return nil, err } for _, entry := range plan.Plan { @@ -778,7 +778,7 @@ func (b *DeploymentBundle) LookupReferencePreDeploy(ctx context.Context, path *s func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey string, entry *deployplan.PlanEntry, errorPrefix string, isPreDeploy bool) bool { sv, ok := b.StateCache.Load(resourceKey) if !ok { - logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no cache entry found for %q", errorPrefix, resourceKey)) + b.deployErrs.add(fmt.Errorf("%s: internal error: no cache entry found for %q", errorPrefix, resourceKey)) return false } @@ -786,7 +786,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st for fieldPathStr, refString := range sv.Refs { refs, ok := dynvar.NewRef(dyn.V(refString)) if !ok { - logdiag.LogError(ctx, fmt.Errorf("%s: cannot parse %q", errorPrefix, refString)) + b.deployErrs.add(fmt.Errorf("%s: cannot parse %q", errorPrefix, refString)) return false } @@ -804,7 +804,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st ref := "${" + pathString + "}" targetPath, err := structpath.ParsePath(pathString) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: cannot parse reference %q: %w", errorPrefix, ref, err)) + b.deployErrs.add(fmt.Errorf("%s: cannot parse reference %q: %w", errorPrefix, ref, err)) return false } @@ -815,20 +815,20 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st if errors.Is(err, errDelayed) { continue } - logdiag.LogError(ctx, fmt.Errorf("%s: cannot resolve %q: %w", errorPrefix, ref, err)) + b.deployErrs.add(fmt.Errorf("%s: cannot resolve %q: %w", errorPrefix, ref, err)) return false } } else { value, err = b.LookupReferencePostDeploy(ctx, targetPath) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: cannot resolve %q: %w", errorPrefix, ref, err)) + b.deployErrs.add(fmt.Errorf("%s: cannot resolve %q: %w", errorPrefix, ref, err)) return false } } err = sv.ResolveRef(ref, value) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: cannot update %s with value of %q: %w", errorPrefix, fieldPathStr, ref, err)) + b.deployErrs.add(fmt.Errorf("%s: cannot update %s with value of %q: %w", errorPrefix, fieldPathStr, ref, err)) return false } resolved = true @@ -838,7 +838,7 @@ func (b *DeploymentBundle) resolveReferences(ctx context.Context, resourceKey st // Sync resolved values back to StructVarJSON for serialization if resolved { if err := sv.SyncToJSON(entry.NewState); err != nil { - logdiag.LogError(ctx, fmt.Errorf("%s: cannot save state: %w", errorPrefix, err)) + b.deployErrs.add(fmt.Errorf("%s: cannot save state: %w", errorPrefix, err)) return false } } diff --git a/bundle/direct/pkg.go b/bundle/direct/pkg.go index 48a9c5a2ff7..741a6f15c50 100644 --- a/bundle/direct/pkg.go +++ b/bundle/direct/pkg.go @@ -2,6 +2,7 @@ package direct import ( "context" + "errors" "fmt" "reflect" "sync" @@ -44,6 +45,36 @@ type DeploymentBundle struct { Plan *deployplan.Plan RemoteStateCache sync.Map StateCache structvar.Cache + + // deployErrs collects errors reported by the parallel graph workers in + // CalculatePlan and Apply. Workers record errors here (thread-safe) and + // signal failure to the graph by returning false; the collected errors are + // joined and returned to the caller once the run completes. + deployErrs errorList +} + +// errorList is a thread-safe accumulator for errors produced by parallel workers. +type errorList struct { + mu sync.Mutex + errs []error +} + +func (e *errorList) reset() { + e.mu.Lock() + defer e.mu.Unlock() + e.errs = nil +} + +func (e *errorList) add(err error) { + e.mu.Lock() + defer e.mu.Unlock() + e.errs = append(e.errs, err) +} + +func (e *errorList) join() error { + e.mu.Lock() + defer e.mu.Unlock() + return errors.Join(e.errs...) } // SetRemoteState updates the remote state with type validation and marks as fresh. diff --git a/bundle/internal/bundletest/benchmark.go b/bundle/internal/bundletest/benchmark.go index 59647d814fd..cc683e49f1c 100644 --- a/bundle/internal/bundletest/benchmark.go +++ b/bundle/internal/bundletest/benchmark.go @@ -231,7 +231,7 @@ func BundleV(b *testing.B, numJobs int) dyn.Value { } // Apply noop mutator to initialize the bundle value. - bundle.ApplyFuncContext(b.Context(), &myBundle, func(ctx context.Context, b *bundle.Bundle) {}) + require.NoError(b, bundle.ApplyFuncContext(b.Context(), &myBundle, func(ctx context.Context, b *bundle.Bundle) {})) return myBundle.Config.Value() } @@ -262,7 +262,7 @@ func Bundle(b *testing.B, numJobs int) *bundle.Bundle { } // Apply noop mutator to initialize the bundle value. - bundle.ApplyFuncContext(b.Context(), &myBundle, func(ctx context.Context, b *bundle.Bundle) {}) + require.NoError(b, bundle.ApplyFuncContext(b.Context(), &myBundle, func(ctx context.Context, b *bundle.Bundle) {})) return &myBundle } diff --git a/bundle/internal/bundletest/mutate.go b/bundle/internal/bundletest/mutate.go index 00adc7d29e0..8a634876a50 100644 --- a/bundle/internal/bundletest/mutate.go +++ b/bundle/internal/bundletest/mutate.go @@ -10,8 +10,8 @@ import ( ) func Mutate(t *testing.T, b *bundle.Bundle, f func(v dyn.Value) (dyn.Value, error)) { - bundle.ApplyFuncContext(t.Context(), b, func(ctx context.Context, b *bundle.Bundle) { + require.NoError(t, bundle.ApplyFuncContext(t.Context(), b, func(ctx context.Context, b *bundle.Bundle) { err := b.Config.Mutate(f) require.NoError(t, err) - }) + })) } diff --git a/bundle/internal/bundletest/mutator_benchmark_test.go b/bundle/internal/bundletest/mutator_benchmark_test.go index 03d63dcd072..031768218d6 100644 --- a/bundle/internal/bundletest/mutator_benchmark_test.go +++ b/bundle/internal/bundletest/mutator_benchmark_test.go @@ -36,7 +36,7 @@ func benchmarkWalkReadOnlyBaseline(b *testing.B, numJobs int) { for b.Loop() { var paths []dyn.Path - bundle.ApplyFuncContext(b.Context(), myBundle, func(ctx context.Context, b *bundle.Bundle) { + _ = bundle.ApplyFuncContext(b.Context(), myBundle, func(ctx context.Context, b *bundle.Bundle) { _ = dyn.WalkReadOnly(b.Config.Value(), func(p dyn.Path, v dyn.Value) error { paths = append(paths, p) return nil @@ -49,7 +49,7 @@ func benchmarkNoopBaseline(b *testing.B, numJobs int) { myBundle := Bundle(b, numJobs) for b.Loop() { - bundle.ApplyFuncContext(b.Context(), myBundle, func(ctx context.Context, b *bundle.Bundle) {}) + _ = bundle.ApplyFuncContext(b.Context(), myBundle, func(ctx context.Context, b *bundle.Bundle) {}) } } diff --git a/bundle/libraries/expand_glob_references.go b/bundle/libraries/expand_glob_references.go index 903cc58f286..5bb7d47d7c1 100644 --- a/bundle/libraries/expand_glob_references.go +++ b/bundle/libraries/expand_glob_references.go @@ -9,6 +9,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/patchwheel" ) @@ -173,7 +174,7 @@ var pipelineEnvDepsPattern = dyn.NewPattern( dyn.Key("dependencies"), ) -func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) error { expanders := []expandPattern{ { pattern: taskLibrariesPattern, @@ -211,10 +212,10 @@ func (e *expand) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { return v, nil }) if err != nil { - diags = diags.Extend(diag.FromErr(err)) + return err } - return diags + return logdiag.Flush(ctx, diags) } func (e *expand) Name() string { diff --git a/bundle/libraries/filer.go b/bundle/libraries/filer.go index 762732262be..84bc686cd8e 100644 --- a/bundle/libraries/filer.go +++ b/bundle/libraries/filer.go @@ -18,7 +18,7 @@ const InternalDirName = ".internal" // Supported locations: // 1. WSFS // 2. UC volumes -func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) { +func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, error) { artifactPath := b.Config.Workspace.ArtifactPath if artifactPath == "" { return nil, "", diag.Errorf("remote artifact path not configured") @@ -36,7 +36,7 @@ func GetFilerForLibraries(ctx context.Context, b *bundle.Bundle) (filer.Filer, s } } -func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, diag.Diagnostics) { +func GetFilerForLibrariesCleanup(ctx context.Context, b *bundle.Bundle) (filer.Filer, string, error) { artifactPath := b.Config.Workspace.ArtifactPath if artifactPath == "" { return nil, "", diag.Errorf("remote artifact path not configured") diff --git a/bundle/libraries/filer_test.go b/bundle/libraries/filer_test.go index f3de3e41e73..f399c9fa323 100644 --- a/bundle/libraries/filer_test.go +++ b/bundle/libraries/filer_test.go @@ -26,7 +26,7 @@ func TestGetFilerForLibrariesValidWsfs(t *testing.T) { b.SetWorkpaceClient(m.WorkspaceClient) client, uploadPath, diags := GetFilerForLibraries(t.Context(), b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, "/Workspace/foo/bar/artifacts/.internal", uploadPath) assert.IsType(t, &filer.WorkspaceFilesClient{}, client) @@ -46,7 +46,7 @@ func TestGetFilerForLibrariesCleanupValidWsfs(t *testing.T) { b.SetWorkpaceClient(m.WorkspaceClient) client, uploadPath, diags := GetFilerForLibrariesCleanup(t.Context(), b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, "/Workspace/foo/bar/artifacts", uploadPath) assert.IsType(t, &filer.WorkspaceFilesClient{}, client) @@ -66,7 +66,7 @@ func TestGetFilerForLibrariesValidUcVolume(t *testing.T) { b.SetWorkpaceClient(m.WorkspaceClient) client, uploadPath, diags := GetFilerForLibraries(t.Context(), b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, "/Volumes/main/my_schema/my_volume/.internal", uploadPath) assert.IsType(t, &filer.FilesClient{}, client) @@ -86,7 +86,7 @@ func TestGetFilerForLibrariesCleanupValidUcVolume(t *testing.T) { b.SetWorkpaceClient(m.WorkspaceClient) client, uploadPath, diags := GetFilerForLibrariesCleanup(t.Context(), b) - require.NoError(t, diags.Error()) + require.NoError(t, diags) assert.Equal(t, "/Volumes/main/my_schema/my_volume", uploadPath) assert.IsType(t, &filer.FilesClient{}, client) @@ -104,5 +104,5 @@ func TestGetFilerForLibrariesRemotePathNotSet(t *testing.T) { b.SetWorkpaceClient(m.WorkspaceClient) _, _, diags := GetFilerForLibraries(t.Context(), b) - require.EqualError(t, diags.Error(), "remote artifact path not configured") + require.EqualError(t, diags, "remote artifact path not configured") } diff --git a/bundle/libraries/filer_volume.go b/bundle/libraries/filer_volume.go index f4b5f51f0c2..8ecfbeaea50 100644 --- a/bundle/libraries/filer_volume.go +++ b/bundle/libraries/filer_volume.go @@ -4,12 +4,11 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" ) -func filerForVolume(ctx context.Context, b *bundle.Bundle, uploadPath string) (filer.Filer, string, diag.Diagnostics) { +func filerForVolume(ctx context.Context, b *bundle.Bundle, uploadPath string) (filer.Filer, string, error) { w := b.WorkspaceClient(ctx) f, err := filer.NewFilesClient(w, uploadPath) - return f, uploadPath, diag.FromErr(err) + return f, uploadPath, err } diff --git a/bundle/libraries/filer_workspace.go b/bundle/libraries/filer_workspace.go index 3d223c342f8..b58e1c01357 100644 --- a/bundle/libraries/filer_workspace.go +++ b/bundle/libraries/filer_workspace.go @@ -4,11 +4,10 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/filer" ) -func filerForWorkspace(ctx context.Context, b *bundle.Bundle, uploadPath string) (filer.Filer, string, diag.Diagnostics) { +func filerForWorkspace(ctx context.Context, b *bundle.Bundle, uploadPath string) (filer.Filer, string, error) { f, err := filer.NewWorkspaceFilesClient(b.WorkspaceClient(ctx), uploadPath) - return f, uploadPath, diag.FromErr(err) + return f, uploadPath, err } diff --git a/bundle/libraries/remote_path.go b/bundle/libraries/remote_path.go index 22784a63358..d8f214b245a 100644 --- a/bundle/libraries/remote_path.go +++ b/bundle/libraries/remote_path.go @@ -9,21 +9,20 @@ import ( "slices" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) // ReplaceWithRemotePath updates all the libraries paths to point to the remote location // where the libraries will be uploaded later. -func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][]LocationToUpdate, diag.Diagnostics) { - _, uploadPath, diags := GetFilerForLibraries(ctx, b) - if diags.HasError() { - return nil, diags +func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][]LocationToUpdate, error) { + _, uploadPath, err := GetFilerForLibraries(ctx, b) + if err != nil { + return nil, err } libs, err := collectLocalLibraries(b) if err != nil { - return nil, diag.FromErr(err) + return nil, err } sources := slices.Sorted(maps.Keys(libs)) @@ -45,10 +44,10 @@ func ReplaceWithRemotePath(ctx context.Context, b *bundle.Bundle) (map[string][] return v, nil }) if err != nil { - diags = diags.Extend(diag.FromErr(err)) + return nil, err } - return libs, diags + return libs, nil } // Collect all libraries from the bundle configuration and their config paths. diff --git a/bundle/libraries/same_name_libraries.go b/bundle/libraries/same_name_libraries.go index 49776fbd8c9..e017940f6a0 100644 --- a/bundle/libraries/same_name_libraries.go +++ b/bundle/libraries/same_name_libraries.go @@ -2,12 +2,15 @@ package libraries import ( "context" + "maps" "path/filepath" + "slices" "strings" "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/logdiag" ) type checkForSameNameLibraries struct{} @@ -28,8 +31,7 @@ type libData struct { otherPaths []string } -func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics +func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) error { libs := make(map[string]*libData) err := b.Config.Mutate(func(rootConfig dyn.Value) (dyn.Value, error) { @@ -76,11 +78,17 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) return rootConfig, nil }) + if err != nil { + return err + } // Iterate over all the libraries and check if there are any duplicates. // Duplicates will have more than one location. // If there are duplicates, add a diagnostic. - for lib, lv := range libs { + // Sort the keys for deterministic output when multiple duplicates exist. + var diags diag.Diagnostics + for _, lib := range slices.Sorted(maps.Keys(libs)) { + lv := libs[lib] if len(lv.locations) > 1 { diags = append(diags, diag.Diagnostic{ Severity: diag.Error, @@ -91,11 +99,8 @@ func (c checkForSameNameLibraries) Apply(ctx context.Context, b *bundle.Bundle) }) } } - if err != nil { - diags = diags.Extend(diag.FromErr(err)) - } - return diags + return logdiag.Flush(ctx, diags) } func (c checkForSameNameLibraries) Name() string { diff --git a/bundle/libraries/switch_to_patched_wheels.go b/bundle/libraries/switch_to_patched_wheels.go index 0a9d1846041..e3b30356772 100644 --- a/bundle/libraries/switch_to_patched_wheels.go +++ b/bundle/libraries/switch_to_patched_wheels.go @@ -8,13 +8,12 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/log" ) type switchToPatchedWheels struct{} -func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (c switchToPatchedWheels) Apply(ctx context.Context, b *bundle.Bundle) error { replacements := getReplacements(ctx, b.Config.Artifacts, b.SyncRoot.Native()) if len(replacements) == 0 { diff --git a/bundle/libraries/upload.go b/bundle/libraries/upload.go index b292fe43b79..c2cce11ca03 100644 --- a/bundle/libraries/upload.go +++ b/bundle/libraries/upload.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" @@ -40,10 +39,10 @@ type LocationToUpdate struct { location dyn.Location } -func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { - client, _, diags := GetFilerForLibraries(ctx, b) - if diags.HasError() { - return diags +func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) error { + client, _, err := GetFilerForLibraries(ctx, b) + if err != nil { + return err } // Only set the filer client if it's not already set. We use the client field @@ -71,10 +70,10 @@ func (u *upload) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { } if err := errs.Wait(); err != nil { - return diag.FromErr(err) + return err } - return diags + return nil } func (u *upload) Name() string { diff --git a/bundle/metrics/track_used_compute.go b/bundle/metrics/track_used_compute.go index 1f3bf051dc0..91fbc32cd9f 100644 --- a/bundle/metrics/track_used_compute.go +++ b/bundle/metrics/track_used_compute.go @@ -4,7 +4,6 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" ) type trackUsedCompute struct{} @@ -13,7 +12,7 @@ func (c *trackUsedCompute) Name() string { return "trackUsedCompute" } -func (c *trackUsedCompute) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (c *trackUsedCompute) Apply(ctx context.Context, b *bundle.Bundle) error { // Track different types of compute used hasServerlessCompute := false hasClassicJobCompute := false diff --git a/bundle/mutator.go b/bundle/mutator.go index 90fdba28c9b..af9809a77ae 100644 --- a/bundle/mutator.go +++ b/bundle/mutator.go @@ -2,6 +2,7 @@ package bundle import ( "context" + "errors" "fmt" "reflect" "time" @@ -19,8 +20,9 @@ type Mutator interface { // Name returns the mutators name. Name() string - // Apply mutates the specified bundle object. - Apply(context.Context, *Bundle) diag.Diagnostics + // Apply mutates the specified bundle object. It returns an error to abort + // the pipeline; warnings and recommendations are emitted via logdiag.LogDiag. + Apply(context.Context, *Bundle) error } // safeMutatorName returns the package name and type name of the underlying mutator type @@ -55,7 +57,10 @@ func safeMutatorName(m Mutator) string { return packageName + ".(" + typeName + ")" } -func ApplyContext(ctx context.Context, b *Bundle, m Mutator) { +// ApplyContext applies a single mutator. It returns the error produced by the +// mutator (if any); warnings and recommendations are emitted directly via +// logdiag.LogDiag and are not part of the returned error. +func ApplyContext(ctx context.Context, b *Bundle, m Mutator) (err error) { t0 := time.Now() defer func() { duration := time.Since(t0).Milliseconds() @@ -75,33 +80,27 @@ func ApplyContext(ctx context.Context, b *Bundle, m Mutator) { log.Debugf(ctx, "Apply") - err := b.Config.MarkMutatorEntry(ctx) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("entry error: %w", err)) - return + if entryErr := b.Config.MarkMutatorEntry(ctx); entryErr != nil { + return fmt.Errorf("entry error: %w", entryErr) } defer func() { - err := b.Config.MarkMutatorExit(ctx) - if err != nil { - logdiag.LogError(ctx, fmt.Errorf("exit error: %w", err)) + if exitErr := b.Config.MarkMutatorExit(ctx); exitErr != nil && err == nil { + err = fmt.Errorf("exit error: %w", exitErr) } }() - diags := m.Apply(ctx, b) - - for _, d := range diags { - logdiag.LogDiag(ctx, d) - } + return m.Apply(ctx, b) } -func ApplySeqContext(ctx context.Context, b *Bundle, mutators ...Mutator) { +// ApplySeqContext applies mutators in order, stopping at the first error. +func ApplySeqContext(ctx context.Context, b *Bundle, mutators ...Mutator) error { for _, m := range mutators { - ApplyContext(ctx, b, m) - if logdiag.HasError(ctx) { - break + if err := ApplyContext(ctx, b, m); err != nil { + return err } } + return nil } type funcMutator struct { @@ -112,18 +111,21 @@ func (m funcMutator) Name() string { return "" } -func (m funcMutator) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { +func (m funcMutator) Apply(ctx context.Context, b *Bundle) error { m.fn(ctx, b) return nil } // ApplyFuncContext applies an inline-specified function mutator. -func ApplyFuncContext(ctx context.Context, b *Bundle, fn func(context.Context, *Bundle)) { - ApplyContext(ctx, b, funcMutator{fn}) +func ApplyFuncContext(ctx context.Context, b *Bundle, fn func(context.Context, *Bundle)) error { + return ApplyContext(ctx, b, funcMutator{fn}) } // Test helpers. TODO: move to separate package. +// Apply is a test helper that runs a mutator and returns the diagnostics it +// produced: warnings/recommendations collected via logdiag plus the returned +// error (if any) as a trailing error diagnostic. func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics { if !logdiag.IsSetup(ctx) { ctx = logdiag.InitContext(ctx) @@ -133,11 +135,18 @@ func Apply(ctx context.Context, b *Bundle, m Mutator) diag.Diagnostics { panic(fmt.Sprintf("Already have %d diags collected: %v", len(previous), previous)) } logdiag.SetCollect(ctx, true) - ApplyContext(ctx, b, m) - return logdiag.FlushCollected(ctx) + err := ApplyContext(ctx, b, m) + diags := logdiag.FlushCollected(ctx) + // A mutator that renders its own diagnostics returns the ErrAlreadyPrinted + // sentinel; those diagnostics are already in Collected, so only append errors + // that were returned directly (not yet rendered). + if err != nil && !errors.Is(err, logdiag.ErrAlreadyPrinted) { + diags = append(diags, diag.DiagnosticFromError(err)) + } + return diags } -// Test helper to get diagnostics in this call +// ApplySeq is a test helper to get diagnostics in this call func ApplySeq(ctx context.Context, b *Bundle, mutators ...Mutator) diag.Diagnostics { if !logdiag.IsSetup(ctx) { ctx = logdiag.InitContext(ctx) @@ -147,6 +156,13 @@ func ApplySeq(ctx context.Context, b *Bundle, mutators ...Mutator) diag.Diagnost panic(fmt.Sprintf("Already have %d diags collected: %v", len(previous), previous)) } logdiag.SetCollect(ctx, true) - ApplySeqContext(ctx, b, mutators...) - return logdiag.FlushCollected(ctx) + err := ApplySeqContext(ctx, b, mutators...) + diags := logdiag.FlushCollected(ctx) + // A mutator that renders its own diagnostics returns the ErrAlreadyPrinted + // sentinel; those diagnostics are already in Collected, so only append errors + // that were returned directly (not yet rendered). + if err != nil && !errors.Is(err, logdiag.ErrAlreadyPrinted) { + diags = append(diags, diag.DiagnosticFromError(err)) + } + return diags } diff --git a/bundle/mutator_read_only.go b/bundle/mutator_read_only.go index b4d55e41f16..ddca865cd8e 100644 --- a/bundle/mutator_read_only.go +++ b/bundle/mutator_read_only.go @@ -2,10 +2,10 @@ package bundle import ( "context" + "errors" "sync" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" ) type ReadOnlyMutator interface { @@ -26,10 +26,11 @@ func (*RO) IsRO() {} // Warning: none of the mutators involved must modify bundle directly or indirectly. In particular, // they must not call bundle.ApplyContext or bundle.ApplyContextSeq because those include writes to config even if mutator does not. // Deprecated: do not use for new use cases. Refactor your parallel task not to depend on bundle at all. -func ApplyParallel(ctx context.Context, b *Bundle, mutators ...ReadOnlyMutator) { +func ApplyParallel(ctx context.Context, b *Bundle, mutators ...ReadOnlyMutator) error { var wg sync.WaitGroup contexts := make([]context.Context, len(mutators)) + errs := make([]error, len(mutators)) for ind, m := range mutators { contexts[ind] = log.NewContext(ctx, log.GetLogger(ctx).With("mutator", m.Name())) //nolint:fatcontext // independent contexts from same parent, not nested @@ -39,13 +40,13 @@ func ApplyParallel(ctx context.Context, b *Bundle, mutators ...ReadOnlyMutator) for ind, m := range mutators { wg.Go(func() { - // We're not using bundle.ApplyContext here because we don't do copy between typed and dynamic values - diags := m.Apply(contexts[ind], b) - for _, d := range diags { - logdiag.LogDiag(ctx, d) - } + // We're not using bundle.ApplyContext here because we don't do copy between typed and dynamic values. + // Mutators emit warnings/recommendations via logdiag.LogDiag and return an error to report failures. + errs[ind] = m.Apply(contexts[ind], b) }) } wg.Wait() + + return errors.Join(errs...) } diff --git a/bundle/mutator_test.go b/bundle/mutator_test.go index e4504d657b0..62998c08437 100644 --- a/bundle/mutator_test.go +++ b/bundle/mutator_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/stretchr/testify/assert" ) @@ -18,9 +17,9 @@ func (t *testMutator) Name() string { return "test" } -func (t *testMutator) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { +func (t *testMutator) Apply(ctx context.Context, b *Bundle) error { t.applyCalled++ - return ApplySeq(ctx, b, t.nestedMutators...) + return ApplySeqContext(ctx, b, t.nestedMutators...) } func TestMutator(t *testing.T) { diff --git a/bundle/permissions/permission_diagnostics.go b/bundle/permissions/permission_diagnostics.go index e25ccd5e724..8ed4f30b8aa 100644 --- a/bundle/permissions/permission_diagnostics.go +++ b/bundle/permissions/permission_diagnostics.go @@ -10,6 +10,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/iamutil" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/set" ) @@ -29,7 +30,7 @@ func (m *permissionDiagnostics) Name() string { return "CheckPermissions" } -func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) error { if len(b.Config.Permissions) == 0 { // Only warn if there is an explicit top-level permissions section return nil @@ -46,7 +47,7 @@ func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) dia identityType = "service_principal_name" } - return diag.Diagnostics{{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Recommendation, Summary: fmt.Sprintf("permissions section should explicitly include the current deployment identity '%s' or one of its groups\n"+ "If it is not included, CAN_MANAGE permissions are only applied if the present identity is used to deploy.\n\n"+ @@ -61,7 +62,8 @@ func (m *permissionDiagnostics) Apply(ctx context.Context, b *bundle.Bundle) dia ), Locations: []dyn.Location{b.Config.GetLocation("permissions")}, ID: diag.PermissionNotIncluded, - }} + }) + return nil } // analyzeBundlePermissions analyzes the top-level permissions of the bundle. diff --git a/bundle/permissions/permission_report.go b/bundle/permissions/permission_report.go index 36526eeeba9..3781b36c270 100644 --- a/bundle/permissions/permission_report.go +++ b/bundle/permissions/permission_report.go @@ -14,7 +14,7 @@ import ( // // Note that since the workspace API doesn't always distinguish between permission denied and path errors, // we must treat this as a "possible permission error". See acquire.go for more about this. -func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path string) diag.Diagnostics { +func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path string) error { log.Errorf(ctx, "Failed to update, encountered possible permission error: %v", path) me := b.Config.Workspace.CurrentUser.User @@ -25,7 +25,7 @@ func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path canManageBundle, assistance := analyzeBundlePermissions(b) if !canManageBundle { - return diag.Diagnostics{{ + return diag.Diagnostic{ Summary: fmt.Sprintf("unable to deploy to %s as %s.\n"+ "Please make sure the current user or one of their groups is listed under the permissions of this bundle.\n"+ "%s\n"+ @@ -34,13 +34,13 @@ func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path path, userName, assistance), Severity: diag.Error, ID: diag.PathPermissionDenied, - }} + } } // According databricks.yml, the current user has the right permissions. // But we're still seeing permission errors. So someone else will need // to redeploy the bundle with the right set of permissions. - return diag.Diagnostics{{ + return diag.Diagnostic{ Summary: fmt.Sprintf("unable to deploy to %s as %s. Cannot apply local deployment permissions.\n"+ "%s\n"+ "They can redeploy the project to apply the latest set of permissions.\n"+ @@ -48,5 +48,5 @@ func ReportPossiblePermissionDenied(ctx context.Context, b *bundle.Bundle, path path, userName, assistance), Severity: diag.Error, ID: diag.CannotChangePathPermissions, - }} + } } diff --git a/bundle/permissions/permission_report_test.go b/bundle/permissions/permission_report_test.go index 52437aacdf8..bd5e71ac30d 100644 --- a/bundle/permissions/permission_report_test.go +++ b/bundle/permissions/permission_report_test.go @@ -18,7 +18,7 @@ func TestPermissionsReportPermissionDeniedWithGroup(t *testing.T) { "For assistance, contact the owners of this project.\n" + "They can redeploy the project to apply the latest set of permissions.\n" + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." - require.ErrorContains(t, diags.Error(), expected) + require.ErrorContains(t, diags, expected) } func TestPermissionsReportPermissionDeniedWithOtherGroup(t *testing.T) { @@ -32,7 +32,7 @@ func TestPermissionsReportPermissionDeniedWithOtherGroup(t *testing.T) { "For assistance, users or groups with appropriate permissions may include: othergroup.\n" + "They may need to redeploy the bundle to apply the new permissions.\n" + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." - require.ErrorContains(t, diags.Error(), expected) + require.ErrorContains(t, diags, expected) } func TestPermissionsReportPermissionDeniedWithoutPermission(t *testing.T) { @@ -46,7 +46,7 @@ func TestPermissionsReportPermissionDeniedWithoutPermission(t *testing.T) { "For assistance, contact the owners of this project.\n" + "They may need to redeploy the bundle to apply the new permissions.\n" + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." - require.ErrorContains(t, diags.Error(), expected) + require.ErrorContains(t, diags, expected) } func TestPermissionsReportPermissionDeniedNilPermission(t *testing.T) { @@ -58,7 +58,7 @@ func TestPermissionsReportPermissionDeniedNilPermission(t *testing.T) { "For assistance, contact the owners of this project.\n" + "They may need to redeploy the bundle to apply the new permissions.\n" + "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions" - require.ErrorContains(t, diags.Error(), expected) + require.ErrorContains(t, diags, expected) } func TestPermissionsReportFindOtherOwners(t *testing.T) { @@ -68,7 +68,7 @@ func TestPermissionsReportFindOtherOwners(t *testing.T) { }) diags := permissions.ReportPossiblePermissionDenied(t.Context(), b, "testpath") - require.ErrorContains(t, diags.Error(), "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n"+ + require.ErrorContains(t, diags, "EPERM3: unable to deploy to testpath as testuser@databricks.com. Cannot apply local deployment permissions.\n"+ "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n"+ "They can redeploy the project to apply the latest set of permissions.\n"+ "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.") diff --git a/bundle/permissions/terraform_errors.go b/bundle/permissions/terraform_errors.go index cc4c9f61424..1677ab19a0e 100644 --- a/bundle/permissions/terraform_errors.go +++ b/bundle/permissions/terraform_errors.go @@ -11,7 +11,7 @@ import ( "github.com/databricks/cli/libs/log" ) -func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, err error) diag.Diagnostics { +func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, err error) error { _, assistance := analyzeBundlePermissions(b) // In a best-effort attempt to provide actionable error messages, we match @@ -35,7 +35,7 @@ func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, er resource = match[2] } - return diag.Diagnostics{{ + return diag.Diagnostic{ Summary: fmt.Sprintf("permission denied creating or updating %s.\n"+ "%s\n"+ "They can redeploy the project to apply the latest set of permissions.\n"+ @@ -43,5 +43,5 @@ func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, er resource, assistance), Severity: diag.Error, ID: diag.ResourcePermissionDenied, - }} + } } diff --git a/bundle/permissions/terraform_errors_test.go b/bundle/permissions/terraform_errors_test.go index 1ad008e251b..53ece1317ea 100644 --- a/bundle/permissions/terraform_errors_test.go +++ b/bundle/permissions/terraform_errors_test.go @@ -21,7 +21,7 @@ func TestTryExtendTerraformPermissionError1(t *testing.T) { "\n"+ " with databricks_pipeline.my_project_pipeline,\n"+ " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() + " 39: }")) expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n" + @@ -43,7 +43,7 @@ func TestTryExtendTerraformPermissionError2(t *testing.T) { "\n"+ " with databricks_pipeline.my_project_pipeline,\n"+ " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() + " 39: }")) expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + "For assistance, users or groups with appropriate permissions may include: alice@databricks.com, bob@databricks.com.\n" + @@ -63,7 +63,7 @@ func TestTryExtendTerraformPermissionError3(t *testing.T) { "\n"+ " with databricks_pipeline.my_project_pipeline,\n"+ " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() + " 39: }")) expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + "For assistance, contact the owners of this project.\n" + @@ -86,7 +86,7 @@ func TestTryExtendTerraformPermissionErrorNotOwner(t *testing.T) { "\n"+ " with databricks_pipeline.my_project_pipeline,\n"+ " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() + " 39: }")) expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + "For assistance, users or groups with appropriate permissions may include: data_team@databricks.com.\n" + diff --git a/bundle/permissions/validate.go b/bundle/permissions/validate.go index dee7326cfa1..1447668c935 100644 --- a/bundle/permissions/validate.go +++ b/bundle/permissions/validate.go @@ -7,6 +7,7 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/libs/diag" + "github.com/databricks/cli/libs/logdiag" ) type validateSharedRootPermissions struct{} @@ -19,18 +20,16 @@ func (*validateSharedRootPermissions) Name() string { return "ValidateSharedRootPermissions" } -func (*validateSharedRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*validateSharedRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) error { if libraries.IsWorkspaceSharedPath(b.Config.Workspace.RootPath) { - return isUsersGroupPermissionSet(b) + isUsersGroupPermissionSet(ctx, b) } return nil } // isUsersGroupPermissionSet checks that top-level permissions set for bundle contain group_name: users with CAN_MANAGE permission. -func isUsersGroupPermissionSet(b *bundle.Bundle) diag.Diagnostics { - var diags diag.Diagnostics - +func isUsersGroupPermissionSet(ctx context.Context, b *bundle.Bundle) { allUsers := false for _, p := range b.Config.Permissions { if p.GroupName == "users" && p.Level == CAN_MANAGE { @@ -40,12 +39,10 @@ func isUsersGroupPermissionSet(b *bundle.Bundle) diag.Diagnostics { } if !allUsers { - diags = diags.Append(diag.Diagnostic{ + logdiag.LogDiag(ctx, diag.Diagnostic{ Severity: diag.Warning, Summary: fmt.Sprintf("the bundle root path %s is writable by all workspace users", b.Config.Workspace.RootPath), Detail: "The bundle is configured to use /Workspace/Shared, which will give read/write access to all users. If this is intentional, add CAN_MANAGE for 'group_name: users' permission to your bundle configuration. If the deployment should be restricted, move it to a restricted folder such as /Workspace/Users/.", }) } - - return diags } diff --git a/bundle/permissions/workspace_root.go b/bundle/permissions/workspace_root.go index 91e21e69a81..fbf5efc0439 100644 --- a/bundle/permissions/workspace_root.go +++ b/bundle/permissions/workspace_root.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" "github.com/databricks/cli/bundle/paths" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/workspace" "golang.org/x/sync/errgroup" ) @@ -28,10 +27,10 @@ func (*workspaceRootPermissions) Name() string { } // Apply implements bundle.Mutator. -func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (*workspaceRootPermissions) Apply(ctx context.Context, b *bundle.Bundle) error { stateFolderPermissions, err := giveAccessForWorkspaceRoot(ctx, b) if err != nil { - return diag.FromErr(err) + return err } recordPermissionMetrics(b, stateFolderPermissions) diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 48ba7755714..16f64db92c6 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -20,16 +20,18 @@ import ( "github.com/databricks/cli/libs/logdiag" ) -func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, engine engine.EngineType) { +func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, engine engine.EngineType) (err error) { log.Info(ctx, "Phase: bind") - bundle.ApplyContext(ctx, b, lock.Acquire()) - if logdiag.HasError(ctx) { - return + if acquireErr := bundle.ApplyContext(ctx, b, lock.Acquire()); acquireErr != nil { + return acquireErr } defer func() { - bundle.ApplyContext(ctx, b, lock.Release(lock.GoalBind)) + err = logdiag.FlushError(ctx, err) + if releaseErr := bundle.ApplyContext(ctx, b, lock.Release(lock.GoalBind)); releaseErr != nil && err == nil { + err = logdiag.FlushError(ctx, releaseErr) + } }() if engine.IsDirect() { @@ -42,10 +44,9 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en resourceKey := fmt.Sprintf("resources.%s.%s", groupName, opts.ResourceKey) _, statePath := b.StateFilenameDirect(ctx) - result, err := b.DeploymentBundle.Bind(ctx, b.WorkspaceClient(ctx), &b.Config, statePath, resourceKey, opts.ResourceId) - if err != nil { - logdiag.LogError(ctx, err) - return + result, bindErr := b.DeploymentBundle.Bind(ctx, b.WorkspaceClient(ctx), &b.Config, statePath, resourceKey, opts.ResourceId) + if bindErr != nil { + return bindErr } // If there are changes and auto-approve is not set, show plan and ask for confirmation @@ -69,42 +70,36 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, en if !cmdio.IsPromptSupported(ctx) { result.Cancel() - logdiag.LogError(ctx, fmt.Errorf("this bind operation requires user confirmation, but the current console does not support prompting.\nTo proceed, use --auto-approve after reviewing the plan above.%s", agent.AgentNotice())) - return + return fmt.Errorf("this bind operation requires user confirmation, but the current console does not support prompting.\nTo proceed, use --auto-approve after reviewing the plan above.%s", agent.AgentNotice()) } - ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") - if err != nil { + ans, askErr := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.") + if askErr != nil { result.Cancel() - logdiag.LogError(ctx, err) - return + return askErr } if !ans { result.Cancel() - logdiag.LogError(ctx, errors.New("import aborted")) - return + return errors.New("import aborted") } } // Finalize: rename temp state to final location - err = result.Finalize() - if err != nil { - logdiag.LogError(ctx, err) - return + if finalizeErr := result.Finalize(); finalizeErr != nil { + return finalizeErr } } else { // Terraform engine: use terraform import - bundle.ApplySeqContext(ctx, b, + if seqErr := bundle.ApplySeqContext(ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Import(opts), - ) - if logdiag.HasError(ctx) { - return + ); seqErr != nil { + return seqErr } } - statemgmt.PushResourcesState(ctx, b, engine) + return statemgmt.PushResourcesState(ctx, b, engine) } func jsonDump(ctx context.Context, v any, field string) string { @@ -116,16 +111,18 @@ func jsonDump(ctx context.Context, v any, field string) string { return string(b) } -func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, resourceKey string, engine engine.EngineType) { +func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, resourceKey string, engine engine.EngineType) (err error) { log.Info(ctx, "Phase: unbind") - bundle.ApplyContext(ctx, b, lock.Acquire()) - if logdiag.HasError(ctx) { - return + if acquireErr := bundle.ApplyContext(ctx, b, lock.Acquire()); acquireErr != nil { + return acquireErr } defer func() { - bundle.ApplyContext(ctx, b, lock.Release(lock.GoalUnbind)) + err = logdiag.FlushError(ctx, err) + if releaseErr := bundle.ApplyContext(ctx, b, lock.Release(lock.GoalUnbind)); releaseErr != nil && err == nil { + err = logdiag.FlushError(ctx, releaseErr) + } }() if engine.IsDirect() { @@ -135,21 +132,18 @@ func Unbind(ctx context.Context, b *bundle.Bundle, bundleType, tfResourceType, r } fullResourceKey := fmt.Sprintf("resources.%s.%s", groupName, resourceKey) _, statePath := b.StateFilenameDirect(ctx) - err := b.DeploymentBundle.Unbind(ctx, statePath, fullResourceKey) - if err != nil { - logdiag.LogError(ctx, err) - return + if unbindErr := b.DeploymentBundle.Unbind(ctx, statePath, fullResourceKey); unbindErr != nil { + return unbindErr } } else { - bundle.ApplySeqContext(ctx, b, + if seqErr := bundle.ApplySeqContext(ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Unbind(bundleType, tfResourceType, resourceKey), - ) - if logdiag.HasError(ctx) { - return + ); seqErr != nil { + return seqErr } } - statemgmt.PushResourcesState(ctx, b, engine) + return statemgmt.PushResourcesState(ctx, b, engine) } diff --git a/bundle/phases/build.go b/bundle/phases/build.go index 5a32435f8f1..d180431de72 100644 --- a/bundle/phases/build.go +++ b/bundle/phases/build.go @@ -11,16 +11,15 @@ import ( "github.com/databricks/cli/bundle/scripts" "github.com/databricks/cli/bundle/trampoline" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" ) type LibLocationMap map[string][]libraries.LocationToUpdate // The build phase builds artifacts. -func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { +func Build(ctx context.Context, b *bundle.Bundle) (LibLocationMap, error) { log.Info(ctx, "Phase: build") - bundle.ApplySeqContext(ctx, b, + if err := bundle.ApplySeqContext(ctx, b, scripts.Execute(config.ScriptPreBuild), artifacts.Build(), scripts.Execute(config.ScriptPostBuild), @@ -39,18 +38,22 @@ func Build(ctx context.Context, b *bundle.Bundle) LibLocationMap { libraries.CheckForSameNameLibraries(), // SwitchToPatchedWheels must be run after ExpandGlobReferences and after build phase because it Artifact.Source and Artifact.Patched populated libraries.SwitchToPatchedWheels(), - ) + ); err != nil { + return nil, err + } - libs, diags := libraries.ReplaceWithRemotePath(ctx, b) - for _, diag := range diags { - logdiag.LogDiag(ctx, diag) + libs, err := libraries.ReplaceWithRemotePath(ctx, b) + if err != nil { + return nil, err } - bundle.ApplyContext(ctx, b, + if err := bundle.ApplyContext(ctx, b, // TransformWheelTask must be run after ReplaceWithRemotePath so we can use correct remote path in the // transformed notebook trampoline.TransformWheelTask(), - ) + ); err != nil { + return nil, err + } - return libs + return libs, nil } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 840c7d821f1..c36208590d1 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -68,7 +68,7 @@ func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.P return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, targetEngine engine.EngineType) { +func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, targetEngine engine.EngineType) error { // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. cmdio.LogString(ctx, "Deploying resources...") @@ -82,43 +82,52 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, ta err error ) if targetEngine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) - state, err = b.DeploymentBundle.StateDB.Finalize(ctx) + applyErr := b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) + var finalizeErr error + state, finalizeErr = b.DeploymentBundle.StateDB.Finalize(ctx) // Capture the finalized state for deploy telemetry. It carries each // resource's state-size in bytes (from the WAL replay Finalize just // did), so telemetry needs no extra read or parse of the state file. b.Metrics.ResourceState = state + err = errors.Join(applyErr, finalizeErr) } else { - bundle.ApplyContext(ctx, b, terraform.Apply()) - state, err = terraform.ParseResourcesState(ctx, b) - } - if err != nil { - logdiag.LogError(ctx, err) + applyErr := bundle.ApplyContext(ctx, b, terraform.Apply()) + var parseErr error + state, parseErr = terraform.ParseResourcesState(ctx, b) + err = errors.Join(applyErr, parseErr) } + // Flush the deploy error now, before the (potentially slow) state push and + // metadata upload below, so the user sees the failure before that work runs. + err = logdiag.FlushError(ctx, err) + // Even if deployment failed, there might be updates in states that we need to upload - statemgmt.PushResourcesState(ctx, b, targetEngine) - if logdiag.HasError(ctx) { - return + if pushErr := statemgmt.PushResourcesState(ctx, b, targetEngine); pushErr != nil { + return errors.Join(err, logdiag.FlushError(ctx, pushErr)) } - bundle.ApplySeqContext(ctx, b, + if seqErr := bundle.ApplySeqContext(ctx, b, statemgmt.Load(state), metadata.Compute(), metadata.Upload(), statemgmt.UploadStateForYamlSync(targetEngine), - ) + ); seqErr != nil { + return errors.Join(err, logdiag.FlushError(ctx, seqErr)) + } - if !logdiag.HasError(ctx) { - cmdio.LogString(ctx, "Deployment complete!") + if err != nil { + return err } + + cmdio.LogString(ctx, "Deployment complete!") + return nil } // uploadLibraries uploads libraries to the workspace. // It also cleans up the artifacts directory and transforms wheel tasks. // It is called by only "bundle deploy". -func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) { - bundle.ApplySeqContext(ctx, b, +func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]libraries.LocationToUpdate) error { + return bundle.ApplySeqContext(ctx, b, artifacts.CleanUp(), libraries.Upload(libs), ) @@ -126,136 +135,118 @@ func uploadLibraries(ctx context.Context, b *bundle.Bundle, libs map[string][]li // The deploy phase deploys artifacts and resources. // If readPlanPath is provided, the plan is loaded from that file instead of being calculated. -func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, engine engine.EngineType, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan) { +func Deploy(ctx context.Context, b *bundle.Bundle, outputHandler sync.OutputHandler, engine engine.EngineType, libs map[string][]libraries.LocationToUpdate, plan *deployplan.Plan) (err error) { log.Info(ctx, "Phase: deploy") // Core mutators that CRUD resources and modify deployment state. These // mutators need informed consent if they are potentially destructive. - bundle.ApplySeqContext(ctx, b, + if seqErr := bundle.ApplySeqContext(ctx, b, scripts.Execute(config.ScriptPreDeploy), lock.Acquire(), - ) - - if logdiag.HasError(ctx) { + ); seqErr != nil { // lock is not acquired here - return + return seqErr } // lock is acquired here defer func() { - bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)) + // Flush the deploy error before releasing the lock so the user sees the + // failure before this final API call runs. + err = logdiag.FlushError(ctx, err) + if releaseErr := bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDeploy)); releaseErr != nil && err == nil { + err = logdiag.FlushError(ctx, releaseErr) + } }() - uploadLibraries(ctx, b, libs) - if logdiag.HasError(ctx) { - return + if uploadErr := uploadLibraries(ctx, b, libs); uploadErr != nil { + return uploadErr } - bundle.ApplySeqContext(ctx, b, + if seqErr := bundle.ApplySeqContext(ctx, b, files.Upload(outputHandler), deploy.StateUpdate(), deploy.StatePush(), permissions.ApplyWorkspaceRootPermissions(), metrics.TrackUsedCompute(), deploy.ResourcePathMkdir(), - ) - - if logdiag.HasError(ctx) { - return + ); seqErr != nil { + return seqErr } planFromFile := plan != nil if plan == nil { - // State is already open for read by process.go (for direct engine) - plan = RunPlan(ctx, b, engine) - } - - // Stop before opening the WAL for write if planning failed. UpgradeToWrite - // writes a WAL header that only deployCore's Finalize commits or discards; - // returning past it without finalizing leaves a header-only WAL behind. - if logdiag.HasError(ctx) { - return + // State is already open for read by process.go (for direct engine). + // Stop before opening the WAL for write if planning failed. UpgradeToWrite + // writes a WAL header that only deployCore's Finalize commits or discards; + // returning past it without finalizing leaves a header-only WAL behind. + var planErr error + plan, planErr = RunPlan(ctx, b, engine) + if planErr != nil { + return planErr + } } if engine.IsDirect() { // Upgrade from read (opened by process.go) to write mode - if err := b.DeploymentBundle.StateDB.UpgradeToWrite(); err != nil { - logdiag.LogError(ctx, err) - return + if upgradeErr := b.DeploymentBundle.StateDB.UpgradeToWrite(); upgradeErr != nil { + return upgradeErr } } if planFromFile { // Initialize DeploymentBundle for applying the loaded plan - err := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan) - if err != nil { - logdiag.LogError(ctx, err) - return + if initErr := b.DeploymentBundle.InitForApply(ctx, b.WorkspaceClient(ctx), plan); initErr != nil { + return initErr } } - // InitForApply receives ctx and could log a diagnostic without returning an - // error, so re-check before deploying. (UpgradeToWrite above takes no ctx and - // thus cannot log, so the earlier check is enough to guard the WAL open.) - if logdiag.HasError(ctx) { - return + haveApproval, approvalErr := approvalForDeploy(ctx, b, plan) + if approvalErr != nil { + return approvalErr } - - haveApproval, err := approvalForDeploy(ctx, b, plan) - if err != nil { - logdiag.LogError(ctx, err) - return - } - if haveApproval { - deployCore(ctx, b, plan, engine) - } else { + if !haveApproval { cmdio.LogString(ctx, "Deployment cancelled!") - return + return nil } - if logdiag.HasError(ctx) { - return + if coreErr := deployCore(ctx, b, plan, engine); coreErr != nil { + return coreErr } - bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPostDeploy)) + return bundle.ApplyContext(ctx, b, scripts.Execute(config.ScriptPostDeploy)) } -func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *deployplan.Plan { +func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) (*deployplan.Plan, error) { if engine.IsDirect() { plan, err := b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), &b.Config) if err != nil { - logdiag.LogError(ctx, err) - return nil + return nil, err } if len(b.Select) > 0 { plan.FilterToSelected(b.Select) } - return plan + return plan, nil } // b.Select is rejected for the terraform engine in ProcessBundleRet, so it is // never set here. - bundle.ApplySeqContext(ctx, b, + if err := bundle.ApplySeqContext(ctx, b, terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("deploy")), - ) - - if logdiag.HasError(ctx) { - return nil + ); err != nil { + return nil, err } tf := b.Terraform if tf == nil { - logdiag.LogError(ctx, errors.New("terraform not initialized")) - return nil + return nil, errors.New("terraform not initialized") } plan, err := terraform.ShowPlanFile(ctx, tf, b.TerraformPlanPath) if err != nil { - logdiag.LogError(ctx, err) - return nil + return nil, err } for _, group := range b.Config.Resources.AllResources() { @@ -269,7 +260,7 @@ func RunPlan(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) *d } } - return plan + return plan, nil } // If there are more than 1 thousand of a resource type, do not diff --git a/bundle/phases/destroy.go b/bundle/phases/destroy.go index fd580f5e971..2d6257494e9 100644 --- a/bundle/phases/destroy.go +++ b/bundle/phases/destroy.go @@ -75,14 +75,19 @@ func approvalForDestroy(ctx context.Context, b *bundle.Bundle, plan *deployplan. return cmdio.AskYesOrNo(ctx, "Would you like to proceed?") } -func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) { +func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, engine engine.EngineType) error { + var applyErr error if engine.IsDirect() { - b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) + applyErr = b.DeploymentBundle.Apply(ctx, b.WorkspaceClient(ctx), plan) } else { // Core destructive mutators for destroy. These require informed user consent. - bundle.ApplyContext(ctx, b, terraform.Apply()) + applyErr = bundle.ApplyContext(ctx, b, terraform.Apply()) } + // Flush the apply error before the (potentially slow) state finalize below, + // so the user sees the failure before that work runs. + applyErr = logdiag.FlushError(ctx, applyErr) + // Flush WAL to local state file before deleting remote files. // Warn instead of hard-error: resources are already deleted, so proceed // with file cleanup regardless of whether state flush succeeds. @@ -95,43 +100,47 @@ func destroyCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, e } } - if logdiag.HasError(ctx) { - return + if applyErr != nil { + return applyErr } - bundle.ApplyContext(ctx, b, files.Delete()) - - if !logdiag.HasError(ctx) { - cmdio.LogString(ctx, "Destroy complete!") + if err := bundle.ApplyContext(ctx, b, files.Delete()); err != nil { + return logdiag.FlushError(ctx, err) } + + cmdio.LogString(ctx, "Destroy complete!") + return nil } // The destroy phase deletes artifacts and resources. -func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { +func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) (err error) { log.Info(ctx, "Phase: destroy") - ok, err := assertRootPathExists(ctx, b) - if err != nil { - logdiag.LogError(ctx, err) - return + ok, existErr := assertRootPathExists(ctx, b) + if existErr != nil { + return existErr } if !ok { cmdio.LogString(ctx, "No active deployment found to destroy!") - return + return nil } - bundle.ApplyContext(ctx, b, lock.Acquire()) - if logdiag.HasError(ctx) { - return + if acquireErr := bundle.ApplyContext(ctx, b, lock.Acquire()); acquireErr != nil { + return acquireErr } defer func() { - bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)) + // Flush the destroy error before releasing the lock so the user sees the + // failure before this final API call runs. + err = logdiag.FlushError(ctx, err) + if releaseErr := bundle.ApplyContext(ctx, b, lock.Release(lock.GoalDestroy)); releaseErr != nil && err == nil { + err = logdiag.FlushError(ctx, releaseErr) + } }() if !engine.IsDirect() { - bundle.ApplySeqContext(ctx, b, + if seqErr := bundle.ApplySeqContext(ctx, b, // We need to resolve artifact variable (how we do it in build phase) // because some of the to-be-destroyed resource might use this variable. // Not resolving might lead to terraform "Reference to undeclared resource" error @@ -141,50 +150,45 @@ func Destroy(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { terraform.Interpolate(), terraform.Write(), terraform.Plan(terraform.PlanGoal("destroy")), - ) - } - - if logdiag.HasError(ctx) { - return + ); seqErr != nil { + return seqErr + } } var plan *deployplan.Plan if engine.IsDirect() { plan, err = b.DeploymentBundle.CalculatePlan(ctx, b.WorkspaceClient(ctx), nil) if err != nil { - logdiag.LogError(ctx, err) - return + return err } } else { tf := b.Terraform if tf == nil { - logdiag.LogError(ctx, errors.New("terraform not initialized")) - return + return errors.New("terraform not initialized") } plan, err = terraform.ShowPlanFile(ctx, tf, b.TerraformPlanPath) if err != nil { - logdiag.LogError(ctx, err) - return + return err } } - hasApproval, err := approvalForDestroy(ctx, b, plan) - if err != nil { - logdiag.LogError(ctx, err) - return + hasApproval, approvalErr := approvalForDestroy(ctx, b, plan) + if approvalErr != nil { + return approvalErr } - if hasApproval { - if engine.IsDirect() { - // Upgrade from read (opened by process.go) to write mode - if err := b.DeploymentBundle.StateDB.UpgradeToWrite(); err != nil { - logdiag.LogError(ctx, err) - return - } - } - destroyCore(ctx, b, plan, engine) - } else { + if !hasApproval { cmdio.LogString(ctx, "Destroy cancelled!") + return nil + } + + if engine.IsDirect() { + // Upgrade from read (opened by process.go) to write mode + if upgradeErr := b.DeploymentBundle.StateDB.UpgradeToWrite(); upgradeErr != nil { + return upgradeErr + } } + + return destroyCore(ctx, b, plan, engine) } diff --git a/bundle/phases/initialize.go b/bundle/phases/initialize.go index 80127843e83..cd714801280 100644 --- a/bundle/phases/initialize.go +++ b/bundle/phases/initialize.go @@ -22,10 +22,10 @@ import ( // The initialize phase fills in defaults and connects to the workspace. // Interpolation of fields referring to the "bundle" and "workspace" keys // happens upon completion of this phase. -func Initialize(ctx context.Context, b *bundle.Bundle) { +func Initialize(ctx context.Context, b *bundle.Bundle) error { log.Info(ctx, "Phase: initialize") - bundle.ApplySeqContext(ctx, b, + return bundle.ApplySeqContext(ctx, b, // Reads (dynamic): resource.*.* // Checks that none of resources.. is nil. Raises error otherwise. validate.AllResourcesHaveValues(), diff --git a/bundle/phases/load.go b/bundle/phases/load.go index 6fa1106b3bf..9c81ad3f0c5 100644 --- a/bundle/phases/load.go +++ b/bundle/phases/load.go @@ -6,35 +6,32 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" ) // The load phase loads configuration from disk and performs // lightweight preprocessing (anything that can be done without network I/O). -func Load(ctx context.Context, b *bundle.Bundle) { +func Load(ctx context.Context, b *bundle.Bundle) error { log.Info(ctx, "Phase: load") - mutator.DefaultMutators(ctx, b) + return mutator.DefaultMutators(ctx, b) } -func LoadDefaultTarget(ctx context.Context, b *bundle.Bundle) { +func LoadDefaultTarget(ctx context.Context, b *bundle.Bundle) error { log.Info(ctx, "Phase: load") - mutator.DefaultMutators(ctx, b) - if logdiag.HasError(ctx) { - return + if err := mutator.DefaultMutators(ctx, b); err != nil { + return err } - bundle.ApplyContext(ctx, b, mutator.SelectDefaultTarget()) + return bundle.ApplyContext(ctx, b, mutator.SelectDefaultTarget()) } -func LoadNamedTarget(ctx context.Context, b *bundle.Bundle, target string) { +func LoadNamedTarget(ctx context.Context, b *bundle.Bundle, target string) error { log.Info(ctx, "Phase: load") - mutator.DefaultMutators(ctx, b) - if logdiag.HasError(ctx) { - return + if err := mutator.DefaultMutators(ctx, b); err != nil { + return err } - bundle.ApplyContext(ctx, b, mutator.SelectTarget(target)) + return bundle.ApplyContext(ctx, b, mutator.SelectTarget(target)) } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index 0af9394f243..290e6276c88 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -18,8 +18,8 @@ import ( // PreDeployChecks is common set of mutators between "bundle plan" and "bundle deploy". // Note, it is not run in "bundle migrate" so it must not modify the config -func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) { - bundle.ApplySeqContext(ctx, b, +func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine engine.EngineType) error { + return bundle.ApplySeqContext(ctx, b, terraform.CheckDashboardsModifiedRemotely(isPlan, engine), resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), diff --git a/bundle/scripts/scripts.go b/bundle/scripts/scripts.go index 4efcd22a4fd..cef62879b55 100644 --- a/bundle/scripts/scripts.go +++ b/bundle/scripts/scripts.go @@ -13,7 +13,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/exec" "github.com/databricks/cli/libs/log" ) @@ -32,7 +31,7 @@ func (m *script) Name() string { return fmt.Sprintf("scripts.%s", m.scriptHook) } -func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *script) Apply(ctx context.Context, b *bundle.Bundle) error { command := getCommand(b, m.scriptHook) if command == "" { log.Debugf(ctx, "No script defined for %s, skipping", m.scriptHook) @@ -41,12 +40,12 @@ func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { executor, err := exec.NewCommandExecutor(b.BundleRootPath) if err != nil { - return diag.FromErr(err) + return err } cmd, err := executeHook(ctx, executor, command) if err != nil { - return diag.FromErr(fmt.Errorf("failed to execute script: %w", err)) + return fmt.Errorf("failed to execute script: %w", err) } cmdio.LogString(ctx, fmt.Sprintf("Executing '%s' script", m.scriptHook)) @@ -67,7 +66,7 @@ func (m *script) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { err = cmd.Wait() if err != nil { - return diag.FromErr(fmt.Errorf("failed to execute script: %w", err)) + return fmt.Errorf("failed to execute script: %w", err) } return nil diff --git a/bundle/set_default.go b/bundle/set_default.go index 3100f67d57c..3ade8994244 100644 --- a/bundle/set_default.go +++ b/bundle/set_default.go @@ -4,9 +4,7 @@ import ( "context" "fmt" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" - "github.com/databricks/cli/libs/logdiag" ) type setDefault struct { @@ -27,8 +25,8 @@ func (m *setDefault) Name() string { return fmt.Sprintf("SetDefaultMutator(%v, %v, %v)", m.pattern, m.key, m.value) } -func (m *setDefault) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { - err := b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { +func (m *setDefault) Apply(ctx context.Context, b *Bundle) error { + return b.Config.Mutate(func(v dyn.Value) (dyn.Value, error) { return dyn.MapByPattern(v, m.pattern, func(p dyn.Path, v dyn.Value) (dyn.Value, error) { _, err := dyn.GetByPath(v, m.key) switch { @@ -39,26 +37,19 @@ func (m *setDefault) Apply(ctx context.Context, b *Bundle) diag.Diagnostics { } }) }) - if err != nil { - return diag.FromErr(err) - } - - return nil } -func SetDefault(ctx context.Context, b *Bundle, pattern string, value any) { +func SetDefault(ctx context.Context, b *Bundle, pattern string, value any) error { pat, err := dyn.NewPatternFromString(pattern) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s: %w", pattern, err)) - return + return fmt.Errorf("internal error: invalid pattern: %s: %w", pattern, err) } pat, key := pat.SplitKey() if pat == nil || key == "" { - logdiag.LogError(ctx, fmt.Errorf("internal error: invalid pattern: %s", pattern)) - return + return fmt.Errorf("internal error: invalid pattern: %s", pattern) } m := SetDefaultMutator(pat, key, value) - ApplyContext(ctx, b, m) + return ApplyContext(ctx, b, m) } diff --git a/bundle/statemgmt/check_running_resources.go b/bundle/statemgmt/check_running_resources.go index 2ad5adf6da8..a666a8860b9 100644 --- a/bundle/statemgmt/check_running_resources.go +++ b/bundle/statemgmt/check_running_resources.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/service/jobs" @@ -34,7 +33,7 @@ func (l *checkRunningResources) Name() string { return "check-running-resources" } -func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) error { if !b.Config.Bundle.Deployment.FailOnActiveRuns { return nil } @@ -47,16 +46,12 @@ func (l *checkRunningResources) Apply(ctx context.Context, b *bundle.Bundle) dia } else { state, err = terraform.ParseResourcesState(ctx, b) if err != nil { - return diag.FromErr(err) + return err } } w := b.WorkspaceClient(ctx) - err = checkAnyResourceRunning(ctx, w, state) - if err != nil { - return diag.FromErr(err) - } - return nil + return checkAnyResourceRunning(ctx, w, state) } func CheckRunningResource(engine engine.EngineType) bundle.Mutator { diff --git a/bundle/statemgmt/state_load.go b/bundle/statemgmt/state_load.go index 573c69126c2..fba7e2da94d 100644 --- a/bundle/statemgmt/state_load.go +++ b/bundle/statemgmt/state_load.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/config/resources" "github.com/databricks/cli/bundle/statemgmt/resourcestate" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" ) @@ -32,18 +31,18 @@ func (l *load) Name() string { return "statemgmt.Load" } -func (l *load) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (l *load) Apply(ctx context.Context, b *bundle.Bundle) error { return applyState(ctx, b, l.state, l.modes) } // applyState merges the exported resource state into the bundle configuration. -func applyState(ctx context.Context, b *bundle.Bundle, state ExportedResourcesMap, modes []LoadMode) diag.Diagnostics { +func applyState(ctx context.Context, b *bundle.Bundle, state ExportedResourcesMap, modes []LoadMode) error { if err := validateLoadedState(state, modes); err != nil { - return diag.FromErr(err) + return err } if err := StateToBundle(ctx, state, &b.Config); err != nil { - return diag.FromErr(err) + return err } // Merge dashboard etags into configuration. diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go index 7e62bb84967..67b7ceca202 100644 --- a/bundle/statemgmt/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -56,19 +56,19 @@ func (s *StateDesc) HasRemoteTerraformState() bool { return false } -func localRead(ctx context.Context, fullPath string, engine engine.EngineType) *StateDesc { +func localRead(ctx context.Context, fullPath string, engine engine.EngineType) (*StateDesc, error) { content, err := os.ReadFile(fullPath) if err != nil { if !errors.Is(err, fs.ErrNotExist) { - logdiag.LogError(ctx, fmt.Errorf("reading %s: %w", filepath.ToSlash(fullPath), err)) + return nil, fmt.Errorf("reading %s: %w", filepath.ToSlash(fullPath), err) } - return nil + return nil, nil } state := &StateDesc{} err = json.Unmarshal(content, state) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("parsing %s: %w", filepath.ToSlash(fullPath), err)) + return nil, fmt.Errorf("parsing %s: %w", filepath.ToSlash(fullPath), err) } state.SourcePath = filepath.ToSlash(fullPath) @@ -76,7 +76,7 @@ func localRead(ctx context.Context, fullPath string, engine engine.EngineType) * state.IsLocal = true // not populating .content, not needed for local - return state + return state, nil } func _filerRead(ctx context.Context, f filer.Filer, path string) (*StateDesc, error) { @@ -106,20 +106,21 @@ func _filerRead(ctx context.Context, f filer.Filer, path string) (*StateDesc, er return state, nil } -func filerRead(ctx context.Context, f filer.Filer, path string, engine engine.EngineType) *StateDesc { +func filerRead(ctx context.Context, f filer.Filer, path string, engine engine.EngineType) (*StateDesc, error) { state, err := _filerRead(ctx, f, path) if err != nil { - logdiag.LogError(ctx, fmt.Errorf("reading %s: %w", path, err)) - } else if state != nil { + return nil, fmt.Errorf("reading %s: %w", path, err) + } + if state != nil { log.Debugf(ctx, "read %s: %s", path, state.String()) state.Engine = engine } - return state + return state, nil } // PullResourcesState determines correct state to use by reading all 4 states (terraform/direct, local/remote). // If state is present and the requested engine disagrees, a warning is issued and the state's engine is used. -func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull, requiredEngine engine.EngineSetting) (context.Context, *StateDesc) { +func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull, requiredEngine engine.EngineSetting) (context.Context, *StateDesc, error) { var err error // We read all 4 possible states: terraform/direct X local/remote and then use env var to validate that correct one is used. @@ -127,10 +128,9 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always _, localPathDirect := b.StateFilenameDirect(ctx) _, localPathTerraform := b.StateFilenameTerraform(ctx) - states := readStates(ctx, b, alwaysPull) - - if logdiag.HasError(ctx) { - return ctx, nil + states, err := readStates(ctx, b, alwaysPull) + if err != nil { + return ctx, nil, err } var winner *StateDesc @@ -151,8 +151,7 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always err = validateStates(states) if err != nil { - logStatesError(ctx, err.Error(), states) - return ctx, winner + return ctx, winner, statesDiag(diag.Error, err.Error(), states) } if requiredEngine.Type != engine.EngineNotSet && requiredEngine.Type != winner.Engine { @@ -171,12 +170,12 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always ctx = useragent.InContext(ctx, "engine", string(winner.Engine)) if len(states) == 0 { - return ctx, winner + return ctx, winner, nil } if winner.IsLocal { // local state is fresh, nothing to do - return ctx, winner + return ctx, winner, nil } if !winner.IsLocal { @@ -191,54 +190,61 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always err := os.MkdirAll(localStateDir, 0o700) if err != nil { - logdiag.LogError(ctx, err) - return ctx, winner + return ctx, winner, err } // TODO: write + rename err = os.WriteFile(localStatePath, winner.Content, 0o600) if err != nil { - logdiag.LogError(ctx, err) - return ctx, winner + return ctx, winner, err } } - return ctx, winner + return ctx, winner, nil } -func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) []*StateDesc { +func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) ([]*StateDesc, error) { var states []*StateDesc remotePathDirect, localPathDirect := b.StateFilenameDirect(ctx) remotePathTerraform, localPathTerraform := b.StateFilenameTerraform(ctx) - if logdiag.HasError(ctx) { - return nil + directLocalState, err := localRead(ctx, localPathDirect, engine.EngineDirect) + if err != nil { + return nil, err + } + terraformLocalState, err := localRead(ctx, localPathTerraform, engine.EngineTerraform) + if err != nil { + return nil, err } - - directLocalState := localRead(ctx, localPathDirect, engine.EngineDirect) - terraformLocalState := localRead(ctx, localPathTerraform, engine.EngineTerraform) if (directLocalState == nil && terraformLocalState == nil) || alwaysPull { f, err := deploy.StateFiler(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return nil + return nil, err } var wg sync.WaitGroup var directRemoteState, terraformRemoteState *StateDesc + var directRemoteErr, terraformRemoteErr error wg.Go(func() { - directRemoteState = filerRead(ctx, f, remotePathDirect, engine.EngineDirect) + directRemoteState, directRemoteErr = filerRead(ctx, f, remotePathDirect, engine.EngineDirect) }) wg.Go(func() { - terraformRemoteState = filerRead(ctx, f, remotePathTerraform, engine.EngineTerraform) + terraformRemoteState, terraformRemoteErr = filerRead(ctx, f, remotePathTerraform, engine.EngineTerraform) }) wg.Wait() + if directRemoteErr != nil { + return nil, directRemoteErr + } + if terraformRemoteErr != nil { + return nil, terraformRemoteErr + } + // find highest serial across all state files // sorting is stable, so initial setting represents preference (later is preferred): states = []*StateDesc{terraformRemoteState, terraformLocalState, directRemoteState, directLocalState} @@ -250,7 +256,7 @@ func readStates(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull) [] return a.Serial - b.Serial }) - return states + return states, nil } func validateStates(states []*StateDesc) error { @@ -286,22 +292,18 @@ func validateStates(states []*StateDesc) error { return nil } -func logStatesError(ctx context.Context, msg string, states []*StateDesc) { - logStatesDiag(ctx, diag.Error, msg, states) -} - func logStatesWarning(ctx context.Context, msg string, states []*StateDesc) { - logStatesDiag(ctx, diag.Warning, msg, states) + logdiag.LogDiag(ctx, statesDiag(diag.Warning, msg, states)) } -func logStatesDiag(ctx context.Context, severity diag.Severity, msg string, states []*StateDesc) { +func statesDiag(severity diag.Severity, msg string, states []*StateDesc) diag.Diagnostic { var stateStrs []string for _, state := range states { stateStrs = append(stateStrs, state.String()) } - logdiag.LogDiag(ctx, diag.Diagnostic{ + return diag.Diagnostic{ Summary: msg, Severity: severity, Detail: "Available state files:\n- " + strings.Join(stateStrs, "\n- "), - }) + } } diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index f098e8a07cc..866fa764b61 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -12,15 +12,13 @@ import ( "github.com/databricks/cli/libs/cmdio" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" - "github.com/databricks/cli/libs/logdiag" ) // PushResourcesState uploads the local state file to the remote location. -func PushResourcesState(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { +func PushResourcesState(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) error { f, err := deploy.StateFiler(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return + return err } var remotePath, localPath string @@ -36,50 +34,46 @@ func PushResourcesState(ctx context.Context, b *bundle.Bundle, engine engine.Eng // The state file can be absent if terraform apply is skipped because // there are no changes to apply in the plan. log.Debugf(ctx, "Local state file does not exist: %s", localPath) - return + return nil } if err != nil { - logdiag.LogError(ctx, err) - return + return err } defer local.Close() // Upload state file from local cache directory to filer. cmdio.LogString(ctx, "Updating deployment state...") - err = f.Write(ctx, remotePath, local, filer.CreateParentDirectories, filer.OverwriteIfExists) - if err != nil { - logdiag.LogError(ctx, err) - } + return f.Write(ctx, remotePath, local, filer.CreateParentDirectories, filer.OverwriteIfExists) } -func BackupRemoteTerraformState(ctx context.Context, b *bundle.Bundle) { +func BackupRemoteTerraformState(ctx context.Context, b *bundle.Bundle) error { f, err := deploy.StateFiler(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return + return err } remotePath, _ := b.StateFilenameTerraform(ctx) reader, err := f.Read(ctx, remotePath) if errors.Is(err, fs.ErrNotExist) { - return + return nil } if err != nil { log.Warnf(ctx, "backing up terraform state: could not read %s: %s", remotePath, err) - return + return nil } backupPath := remotePath + ".backup" err = f.Write(ctx, backupPath, reader) if err != nil { log.Warnf(ctx, "backing up terraform state: could not write %s: %s", backupPath, err) - return + return nil } err = f.Delete(ctx, remotePath) if err != nil { log.Warnf(ctx, "backing up terraform state: could not delete %s: %s", remotePath, err) } + return nil } diff --git a/bundle/statemgmt/upload_state_for_yaml_sync.go b/bundle/statemgmt/upload_state_for_yaml_sync.go index 163c9fb4fdb..071763e31cd 100644 --- a/bundle/statemgmt/upload_state_for_yaml_sync.go +++ b/bundle/statemgmt/upload_state_for_yaml_sync.go @@ -18,7 +18,6 @@ import ( "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/bundle/env" "github.com/databricks/cli/bundle/migrate" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/filer" @@ -42,7 +41,7 @@ func (m *uploadStateForYamlSync) Name() string { return "statemgmt.UploadStateForYamlSync" } -func (m *uploadStateForYamlSync) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *uploadStateForYamlSync) Apply(ctx context.Context, b *bundle.Bundle) error { if m.engine.IsDirect() { return nil } @@ -141,9 +140,8 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring // the migrated state and config agree on .permissions entries. - bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) - if logdiag.HasError(ctx) { - return false, errors.New("failed to apply secret scope fixups") + if err := bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)); err != nil { + return false, err } // b.Config has been modified by terraform.Interpolate which converts bundle-style @@ -180,12 +178,6 @@ func (m *uploadStateForYamlSync) convertState(ctx context.Context, b *bundle.Bun return false, err } - // Apply reports failures via logdiag instead of returning an error. Don't - // upload a snapshot that is missing entries for the failed resources. - if logdiag.HasError(ctx) { - return false, errors.New("state conversion failed") - } - return true, nil } diff --git a/bundle/tests/include_test.go b/bundle/tests/include_test.go index 50bf177fdfc..88f806ad35a 100644 --- a/bundle/tests/include_test.go +++ b/bundle/tests/include_test.go @@ -18,10 +18,9 @@ func TestIncludeInvalid(t *testing.T) { logdiag.SetCollect(ctx, true) b, err := bundle.Load(ctx, "./include_invalid") require.NoError(t, err) - phases.Load(ctx, b) - diags := logdiag.FlushCollected(ctx) - require.Error(t, diags.Error()) - assert.ErrorContains(t, diags.Error(), "notexists.yml defined in 'include' section does not match any files") + err = phases.Load(ctx, b) + require.Error(t, err) + assert.ErrorContains(t, err, "notexists.yml defined in 'include' section does not match any files") } func TestIncludeWithGlob(t *testing.T) { diff --git a/bundle/tests/loader.go b/bundle/tests/loader.go index 1177190790c..5b7903af232 100644 --- a/bundle/tests/loader.go +++ b/bundle/tests/loader.go @@ -1,6 +1,7 @@ package config_tests import ( + "errors" "testing" "github.com/databricks/cli/bundle/config/mutator/resourcemutator" @@ -18,9 +19,10 @@ func load(t *testing.T, path string) *bundle.Bundle { logdiag.SetCollect(ctx, true) b, err := bundle.Load(ctx, path) require.NoError(t, err) - phases.Load(ctx, b) + loadErr := phases.Load(ctx, b) diags := logdiag.FlushCollected(ctx) require.NoError(t, diags.Error()) + require.NoError(t, loadErr) return b } @@ -38,8 +40,11 @@ func loadTargetWithDiags(t *testing.T, path, env string) (*bundle.Bundle, diag.D return nil, diag.FromErr(err) } - phases.LoadNamedTarget(ctx, b, env) + loadErr := phases.LoadNamedTarget(ctx, b, env) diags := logdiag.FlushCollected(ctx) + if loadErr != nil && !errors.Is(loadErr, logdiag.ErrAlreadyPrinted) { + diags = diags.Append(diag.DiagnosticFromError(loadErr)) + } diags = diags.Extend(bundle.ApplySeq(ctx, b, mutator.RewriteSyncPaths(), diff --git a/bundle/tests/validate_test.go b/bundle/tests/validate_test.go index e10f240aaff..c3124ee10bd 100644 --- a/bundle/tests/validate_test.go +++ b/bundle/tests/validate_test.go @@ -1,6 +1,7 @@ package config_tests import ( + "errors" "testing" "github.com/databricks/cli/bundle" @@ -132,8 +133,11 @@ func TestValidateUniqueResourceIdentifiers(t *testing.T) { require.NoError(t, err) // The UniqueResourceKeys mutator is run as part of the Load phase. - phases.Load(ctx, b) + loadErr := phases.Load(ctx, b) diags := logdiag.FlushCollected(ctx) + if loadErr != nil && !errors.Is(loadErr, logdiag.ErrAlreadyPrinted) { + diags = append(diags, diag.DiagnosticFromError(loadErr)) + } assert.Equal(t, tc.diagnostics, diags) }) } diff --git a/bundle/trampoline/python_dbr_warning.go b/bundle/trampoline/python_dbr_warning.go index 4cc7a67dc8c..cd8e7e6c6e1 100644 --- a/bundle/trampoline/python_dbr_warning.go +++ b/bundle/trampoline/python_dbr_warning.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn/dynvar" "github.com/databricks/cli/libs/log" + "github.com/databricks/cli/libs/logdiag" "github.com/databricks/databricks-sdk-go" "golang.org/x/mod/semver" ) @@ -22,22 +23,27 @@ func WrapperWarning() bundle.Mutator { return &wrapperWarning{} } -func (m *wrapperWarning) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *wrapperWarning) Apply(ctx context.Context, b *bundle.Bundle) error { if isPythonWheelWrapperOn(b) { if config.IsExplicitlyEnabled(b.Config.Presets.SourceLinkedDeployment) { - return diag.Warningf("Python wheel notebook wrapper is not available when using source-linked deployment mode. You can disable this mode by setting 'presets.source_linked_deployment: false'") + logdiag.LogDiag(ctx, diag.Diagnostic{ + Severity: diag.Warning, + Summary: "Python wheel notebook wrapper is not available when using source-linked deployment mode. You can disable this mode by setting 'presets.source_linked_deployment: false'", + }) } return nil } diags := hasIncompatibleWheelTasks(ctx, b) - if len(diags) > 0 { - diags = append(diags, diag.Diagnostic{ - Severity: diag.Error, - Summary: "Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.", - }) + if len(diags) == 0 { + return nil } - return diags + + diags = append(diags, diag.Diagnostic{ + Severity: diag.Error, + Summary: "Python wheel tasks require compute with DBR 13.3+ to include local libraries. Please change your cluster configuration or use the experimental 'python_wheel_wrapper' setting. See https://docs.databricks.com/dev-tools/bundles/python-wheel.html for more information.", + }) + return logdiag.Flush(ctx, diags) } func isPythonWheelWrapperOn(b *bundle.Bundle) bool { diff --git a/bundle/trampoline/python_wheel.go b/bundle/trampoline/python_wheel.go index 722a0b35e6c..dbcfea1fd00 100644 --- a/bundle/trampoline/python_wheel.go +++ b/bundle/trampoline/python_wheel.go @@ -10,7 +10,6 @@ import ( "github.com/databricks/cli/bundle" "github.com/databricks/cli/bundle/libraries" "github.com/databricks/cli/bundle/metrics" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/compute" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -75,20 +74,18 @@ func TransformWheelTask() bundle.Mutator { return transformWheelTask{} } -func (transformWheelTask) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (transformWheelTask) Apply(ctx context.Context, b *bundle.Bundle) error { isEnabled := b.Config.Experimental != nil && b.Config.Experimental.PythonWheelWrapper b.Metrics.AddBoolValue(metrics.ExperimentalPythonWheelWrapperIsSet, isEnabled) if !isEnabled { return nil } - bundle.ApplyContext(ctx, b, NewTrampoline( + return bundle.ApplyContext(ctx, b, NewTrampoline( "python_wheel", &pythonTrampoline{}, NOTEBOOK_TEMPLATE, )) - - return nil } type pythonTrampoline struct{} diff --git a/bundle/trampoline/trampoline.go b/bundle/trampoline/trampoline.go index 600ce3d9c64..6dccaa086f4 100644 --- a/bundle/trampoline/trampoline.go +++ b/bundle/trampoline/trampoline.go @@ -9,7 +9,6 @@ import ( "text/template" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/service/jobs" ) @@ -42,12 +41,12 @@ func (m *trampoline) Name() string { return fmt.Sprintf("trampoline(%s)", m.name) } -func (m *trampoline) Apply(ctx context.Context, b *bundle.Bundle) diag.Diagnostics { +func (m *trampoline) Apply(ctx context.Context, b *bundle.Bundle) error { tasks := m.functions.GetTasks(b) for _, task := range tasks { err := m.generateNotebookWrapper(ctx, b, task) if err != nil { - return diag.FromErr(err) + return err } } return nil diff --git a/cmd/apps/deploy_bundle.go b/cmd/apps/deploy_bundle.go index 7a893e23263..1467c7299ea 100644 --- a/cmd/apps/deploy_bundle.go +++ b/cmd/apps/deploy_bundle.go @@ -89,7 +89,10 @@ func BundleDeployOverrideWithWrapper(wrapError ErrorWrapper) func(*cobra.Command originalRunE := deployCmd.RunE deployCmd.RunE = func(cmd *cobra.Command, args []string) error { if len(args) == 0 { - b := root.TryConfigureBundle(cmd) + b, err := root.TryConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(cmd.Context(), err) + } if b != nil { return runBundleDeploy(cmd, opts) } diff --git a/cmd/apps/dev.go b/cmd/apps/dev.go index 4fb86527c25..68c6625dbd6 100644 --- a/cmd/apps/dev.go +++ b/cmd/apps/dev.go @@ -47,13 +47,15 @@ func detectAppNameFromBundle(cmd *cobra.Command) string { ctx := cmd.Context() // Try to configure bundle (returns nil if no bundle found) - b := root.TryConfigureBundle(cmd) - if b == nil { + b, err := root.TryConfigureBundle(cmd) + if err != nil || b == nil { return "" } // Run initialization to resolve variables, apply prefixes, etc. - phases.Initialize(ctx, b) + if err := phases.Initialize(ctx, b); err != nil { + return "" + } // Check for apps in the bundle bundleApps := b.Config.Resources.Apps diff --git a/cmd/apps/import.go b/cmd/apps/import.go index f22693277ce..07bc19d2315 100644 --- a/cmd/apps/import.go +++ b/cmd/apps/import.go @@ -30,7 +30,6 @@ import ( "github.com/databricks/cli/libs/dyn/convert" "github.com/databricks/cli/libs/dyn/yamlsaver" "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/textutil" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/apps" @@ -335,14 +334,13 @@ func runImport(ctx context.Context, w *databricks.WorkspaceClient, appName, outp if !ok { tfName = resource.ResourceDescription().PluralName } - phases.Bind(ctx, b, &terraform.BindOptions{ + if err := phases.Bind(ctx, b, &terraform.BindOptions{ AutoApprove: true, ResourceType: tfName, ResourceKey: appKey, ResourceId: app.Name, - }, stateDesc.Engine) - if logdiag.HasError(ctx) { - return errors.New("failed to bind resource") + }, stateDesc.Engine); err != nil { + return root.RenderAndReturnError(ctx, err) } if !quiet { diff --git a/cmd/bundle/debug/list_targets.go b/cmd/bundle/debug/list_targets.go index 6349ba46725..4b6d47d2ac0 100644 --- a/cmd/bundle/debug/list_targets.go +++ b/cmd/bundle/debug/list_targets.go @@ -69,14 +69,16 @@ func NewListTargetsCommand() *cobra.Command { cmd.SetContext(ctx) logdiag.SetSeverity(ctx, diag.Warning) - b := bundle.MustLoad(ctx) - if b == nil || logdiag.HasError(ctx) { + b, err := bundle.MustLoad(ctx) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } - phases.Load(ctx, b) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := phases.Load(ctx, b); err != nil { + return root.RenderAndReturnError(ctx, err) } targets := collectTargets(b.Config.Targets) diff --git a/cmd/bundle/deployment/bind_resource.go b/cmd/bundle/deployment/bind_resource.go index fc972d4d7c6..3a458a27f8d 100644 --- a/cmd/bundle/deployment/bind_resource.go +++ b/cmd/bundle/deployment/bind_resource.go @@ -9,7 +9,6 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -47,14 +46,13 @@ func BindResource(cmd *cobra.Command, resourceKey, resourceId string, autoApprov if !ok { tfName = resource.ResourceDescription().PluralName } - phases.Bind(ctx, b, &terraform.BindOptions{ + if err := phases.Bind(ctx, b, &terraform.BindOptions{ AutoApprove: autoApprove, ResourceType: tfName, ResourceKey: resourceKey, ResourceId: resourceId, - }, stateDesc.Engine) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + }, stateDesc.Engine); err != nil { + return root.RenderAndReturnError(ctx, err) } cmdio.LogString(ctx, fmt.Sprintf("Successfully bound %s with an id '%s'", resource.ResourceDescription().SingularName, resourceId)) diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 39d9a0454d5..f6729d02bf1 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -17,6 +17,7 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/env" "github.com/databricks/cli/libs/logdiag" "github.com/databricks/cli/libs/shellquote" @@ -158,9 +159,8 @@ To start using direct engine, set "engine: direct" under bundle in your databric // Apply SecretScopeFixups so the config matches what the direct engine expects. // This adds MANAGE ACL for the current user to all secret scopes, ensuring // the migrated state and config agree on .permissions entries. - bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := bundle.ApplyContext(ctx, b, resourcemutator.SecretScopeFixups(engine.EngineDirect)); err != nil { + return root.RenderAndReturnError(ctx, err) } adapters, err := dresources.InitAll(nil) @@ -177,11 +177,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric } if _, err := stateDB.Finalize(ctx); err != nil { - logdiag.LogError(ctx, err) - } - if logdiag.HasError(ctx) { - logdiag.LogError(ctx, errors.New("migration failed; ensure you have done full deploy before the migration")) - return root.ErrAlreadyPrinted + return root.RenderAndReturnError(ctx, errors.Join(err, errors.New("migration failed; ensure you have done full deploy before the migration"))) } if err := os.Rename(tempStatePath, localPath); err != nil { @@ -194,7 +190,7 @@ To start using direct engine, set "engine: direct" under bundle in your databric err = os.Rename(localTerraformPath, localTerraformBackupPath) if err != nil { // not fatal, since we've increased serial - logdiag.LogError(ctx, err) + logdiag.LogDiag(ctx, diag.DiagnosticFromError(err)) } cmdio.LogString(ctx, fmt.Sprintf(`Success! Migrated %d resources to direct engine state file: %s diff --git a/cmd/bundle/deployment/unbind.go b/cmd/bundle/deployment/unbind.go index 576c46cba0f..6c4617ea467 100644 --- a/cmd/bundle/deployment/unbind.go +++ b/cmd/bundle/deployment/unbind.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/bundle/phases" "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -73,9 +72,8 @@ To re-bind the resource later, use: if !ok { tfName = rd.PluralName } - phases.Unbind(ctx, b, rd.SingularName, tfName, args[0], stateDesc.Engine) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := phases.Unbind(ctx, b, rd.SingularName, tfName, args[0], stateDesc.Engine); err != nil { + return root.RenderAndReturnError(ctx, err) } return nil } diff --git a/cmd/bundle/destroy.go b/cmd/bundle/destroy.go index a4f85667a27..835457d17fb 100644 --- a/cmd/bundle/destroy.go +++ b/cmd/bundle/destroy.go @@ -72,9 +72,8 @@ func CommandBundleDestroy(cmd *cobra.Command, args []string, autoApprove, forceD SkipInitContext: skipInitContext, AlwaysPull: true, PostStateFunc: func(ctx context.Context, b *bundle.Bundle, stateDesc *statemgmt.StateDesc) error { - phases.Destroy(ctx, b, stateDesc.Engine) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := phases.Destroy(ctx, b, stateDesc.Engine); err != nil { + return root.RenderAndReturnError(ctx, err) } return nil }, diff --git a/cmd/bundle/generate/alert.go b/cmd/bundle/generate/alert.go index e80e559f319..ac67b9ccade 100644 --- a/cmd/bundle/generate/alert.go +++ b/cmd/bundle/generate/alert.go @@ -65,8 +65,11 @@ After generation, you can deploy this alert to other targets using: ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } diff --git a/cmd/bundle/generate/app.go b/cmd/bundle/generate/app.go index 120c405e793..d724aaec22b 100644 --- a/cmd/bundle/generate/app.go +++ b/cmd/bundle/generate/app.go @@ -65,8 +65,11 @@ per target environment.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } diff --git a/cmd/bundle/generate/dashboard.go b/cmd/bundle/generate/dashboard.go index 4b73372342b..036fa190164 100644 --- a/cmd/bundle/generate/dashboard.go +++ b/cmd/bundle/generate/dashboard.go @@ -71,7 +71,7 @@ type dashboard struct { err io.Writer } -func (d *dashboard) resolveID(ctx context.Context, b *bundle.Bundle) string { +func (d *dashboard) resolveID(ctx context.Context, b *bundle.Bundle) (string, error) { switch { case d.existingPath != "": return d.resolveFromPath(ctx, b) @@ -79,22 +79,20 @@ func (d *dashboard) resolveID(ctx context.Context, b *bundle.Bundle) string { return d.resolveFromID(ctx, b) } - logdiag.LogError(ctx, errors.New("expected one of --existing-path, --existing-id")) - return "" + return "", errors.New("expected one of --existing-path, --existing-id") } -func (d *dashboard) resolveFromPath(ctx context.Context, b *bundle.Bundle) string { +func (d *dashboard) resolveFromPath(ctx context.Context, b *bundle.Bundle) (string, error) { w := b.WorkspaceClient(ctx) obj, err := w.Workspace.GetStatusByPath(ctx, d.existingPath) if err != nil { if apierr.IsMissing(err) { - logdiag.LogError(ctx, fmt.Errorf("dashboard %q not found", path.Base(d.existingPath))) - return "" + return "", fmt.Errorf("dashboard %q not found", path.Base(d.existingPath)) } // Emit a more descriptive error message for legacy dashboards. if errors.Is(err, apierr.ErrBadRequest) && strings.HasPrefix(err.Error(), "dbsqlDashboard ") { - logdiag.LogDiag(ctx, diag.Diagnostic{ + return "", diag.Diagnostic{ Severity: diag.Error, Summary: fmt.Sprintf("dashboard %q is a legacy dashboard", path.Base(d.existingPath)), Detail: "" + @@ -102,47 +100,41 @@ func (d *dashboard) resolveFromPath(ctx context.Context, b *bundle.Bundle) strin "\n" + "Instructions on how to convert a legacy dashboard to an AI/BI dashboard\n" + "can be found at: https://docs.databricks.com/en/dashboards/clone-legacy-to-aibi.html.", - }) - return "" + } } - logdiag.LogError(ctx, err) - return "" + return "", err } if obj.ObjectType != workspace.ObjectTypeDashboard { found := strings.ToLower(obj.ObjectType.String()) - logdiag.LogDiag(ctx, diag.Diagnostic{ + return "", diag.Diagnostic{ Severity: diag.Error, Summary: "expected a dashboard, found a " + found, - }) - return "" + } } if obj.ResourceId == "" { - logdiag.LogDiag(ctx, diag.Diagnostic{ + return "", diag.Diagnostic{ Severity: diag.Error, Summary: "expected a non-empty dashboard resource ID", - }) - return "" + } } - return obj.ResourceId + return obj.ResourceId, nil } -func (d *dashboard) resolveFromID(ctx context.Context, b *bundle.Bundle) string { +func (d *dashboard) resolveFromID(ctx context.Context, b *bundle.Bundle) (string, error) { w := b.WorkspaceClient(ctx) obj, err := w.Lakeview.GetByDashboardId(ctx, d.existingID) if err != nil { if apierr.IsMissing(err) { - logdiag.LogError(ctx, fmt.Errorf("dashboard with ID %s not found", d.existingID)) - return "" + return "", fmt.Errorf("dashboard with ID %s not found", d.existingID) } - logdiag.LogError(ctx, err) - return "" + return "", err } - return obj.DashboardId + return obj.DashboardId, nil } func remarshalJSON(data []byte) ([]byte, error) { @@ -254,19 +246,17 @@ func (d *dashboard) saveConfiguration(ctx context.Context, b *bundle.Bundle, das return nil } -func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboard *dashboards.Dashboard) { +func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboard *dashboards.Dashboard) error { // Compute [time.Time] for the most recent update. tref, err := time.Parse(time.RFC3339, dashboard.UpdateTime) if err != nil { - logdiag.LogError(ctx, err) - return + return err } for { obj, err := w.Workspace.GetStatusByPath(ctx, dashboard.Path) if err != nil { - logdiag.LogError(ctx, err) - return + return err } // Compute [time.Time] from timestamp in millis since epoch. @@ -277,22 +267,22 @@ func waitForChanges(ctx context.Context, w *databricks.WorkspaceClient, dashboar select { case <-ctx.Done(): - return + return nil case <-time.After(1 * time.Second): } } + + return nil } -func (d *dashboard) updateDashboardForResource(ctx context.Context, b *bundle.Bundle) { +func (d *dashboard) updateDashboardForResource(ctx context.Context, b *bundle.Bundle) error { resource, ok := b.Config.Resources.Dashboards[d.resource] if !ok { - logdiag.LogError(ctx, fmt.Errorf("dashboard resource %q is not defined", d.resource)) - return + return fmt.Errorf("dashboard resource %q is not defined", d.resource) } if resource.FilePath == "" { - logdiag.LogError(ctx, fmt.Errorf("dashboard resource %q has no file path defined", d.resource)) - return + return fmt.Errorf("dashboard resource %q has no file path defined", d.resource) } // Resolve the dashboard ID from the resource. @@ -308,21 +298,19 @@ func (d *dashboard) updateDashboardForResource(ctx context.Context, b *bundle.Bu for { dashboard, err := w.Lakeview.GetByDashboardId(ctx, dashboardID) if err != nil { - logdiag.LogError(ctx, err) - return + return err } if etag != dashboard.Etag { err = d.saveSerializedDashboard(ctx, b, dashboard, dashboardPath) if err != nil { - logdiag.LogError(ctx, err) - return + return err } } // Abort if we are not watching for changes. if !d.watch { - return + return nil } // Update the etag for the next iteration. @@ -332,35 +320,37 @@ func (d *dashboard) updateDashboardForResource(ctx context.Context, b *bundle.Bu // This is much more efficient than polling the dashboard API because it // includes the entire serialized dashboard whereas we're only interested // in the last modified time of the dashboard here. - waitForChanges(ctx, w, dashboard) + if err := waitForChanges(ctx, w, dashboard); err != nil { + return err + } } } -func (d *dashboard) generateForExisting(ctx context.Context, b *bundle.Bundle, dashboardID string) { +func (d *dashboard) generateForExisting(ctx context.Context, b *bundle.Bundle, dashboardID string) error { w := b.WorkspaceClient(ctx) dashboard, err := w.Lakeview.GetByDashboardId(ctx, dashboardID) if err != nil { - logdiag.LogError(ctx, err) - return + return err } key := textutil.NormalizeString(dashboard.DisplayName) err = d.saveConfiguration(ctx, b, dashboard, key) if err != nil { - logdiag.LogError(ctx, err) + return err } if d.bind { err = deployment.BindResource(d.cmd, key, dashboardID, true, false, true) if err != nil { - logdiag.LogError(ctx, err) - return + return err } cmdio.LogString(ctx, fmt.Sprintf("Successfully bound dashboard with an id '%s'", dashboardID)) } + + return nil } -func (d *dashboard) initialize(ctx context.Context, b *bundle.Bundle) { +func (d *dashboard) initialize(ctx context.Context, b *bundle.Bundle) error { // Make the paths absolute if they aren't already. if !filepath.IsAbs(d.resourceDir) { d.resourceDir = filepath.Join(b.BundleRootPath, d.resourceDir) @@ -372,88 +362,84 @@ func (d *dashboard) initialize(ctx context.Context, b *bundle.Bundle) { // Make sure we know how the dashboard path is relative to the resource path. rel, err := filepath.Rel(d.resourceDir, d.dashboardDir) if err != nil { - logdiag.LogError(ctx, err) - return + return err } d.relativeDashboardDir = filepath.ToSlash(rel) + return nil } -func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) { - phases.Initialize(ctx, b) - if logdiag.HasError(ctx) { - return +func (d *dashboard) runForResource(ctx context.Context, b *bundle.Bundle) error { + if err := phases.Initialize(ctx, b); err != nil { + return err } requiredEngine, err := utils.ResolveEngineSetting(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return + return err } - ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) - if logdiag.HasError(ctx) { - return + ctx, stateDesc, err := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if err != nil { + return err } var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { - logdiag.LogError(ctx, err) - return + return err } state = b.DeploymentBundle.ExportState(ctx) } else { - var err error state, err = terraform.ParseResourcesState(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return + return err } } - bundle.ApplySeqContext(ctx, b, + if err := bundle.ApplySeqContext(ctx, b, statemgmt.Load(state), - ) - if logdiag.HasError(ctx) { - return + ); err != nil { + return err } - d.updateDashboardForResource(ctx, b) + return d.updateDashboardForResource(ctx, b) } -func (d *dashboard) runForExisting(ctx context.Context, b *bundle.Bundle) { +func (d *dashboard) runForExisting(ctx context.Context, b *bundle.Bundle) error { // Resolve the ID of the dashboard to generate configuration for. - dashboardID := d.resolveID(ctx, b) - if logdiag.HasError(ctx) { - return + dashboardID, err := d.resolveID(ctx, b) + if err != nil { + return err } - d.generateForExisting(ctx, b, dashboardID) + return d.generateForExisting(ctx, b, dashboardID) } func (d *dashboard) RunE(cmd *cobra.Command, args []string) error { ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } - d.initialize(ctx, b) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := d.initialize(ctx, b); err != nil { + return root.RenderAndReturnError(ctx, err) } if d.resource != "" { - d.runForResource(ctx, b) + err = d.runForResource(ctx, b) } else { - d.runForExisting(ctx, b) + err = d.runForExisting(ctx, b) } - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err != nil { + return root.RenderAndReturnError(ctx, err) } return nil @@ -466,8 +452,8 @@ func filterDashboards(ref resources.Reference) bool { // dashboardResourceCompletion executes to autocomplete the argument to the resource flag. func dashboardResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/bundle/generate/dashboard_test.go b/cmd/bundle/generate/dashboard_test.go index d7cb53ad46b..4d1db5810da 100644 --- a/cmd/bundle/generate/dashboard_test.go +++ b/cmd/bundle/generate/dashboard_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/logdiag" + "github.com/databricks/cli/libs/diag" "github.com/databricks/databricks-sdk-go/apierr" "github.com/databricks/databricks-sdk-go/experimental/mocks" @@ -35,15 +35,11 @@ func TestDashboard_ErrorOnLegacyDashboard(t *testing.T) { }) ctx := t.Context() - ctx = logdiag.InitContext(ctx) - logdiag.SetCollect(ctx, true) b := &bundle.Bundle{} b.SetWorkpaceClient(m.WorkspaceClient) - id := d.resolveID(ctx, b) + id, err := d.resolveID(ctx, b) assert.Empty(t, id) - - diags := logdiag.FlushCollected(ctx) - require.Len(t, diags, 1) - assert.Equal(t, "dashboard \"legacy dashboard\" is a legacy dashboard", diags[0].Summary) + require.Error(t, err) + assert.Equal(t, "dashboard \"legacy dashboard\" is a legacy dashboard", diag.DiagnosticFromError(err).Summary) } diff --git a/cmd/bundle/generate/genie_space.go b/cmd/bundle/generate/genie_space.go index 649f0cacc2c..6d57d1ed8d3 100644 --- a/cmd/bundle/generate/genie_space.go +++ b/cmd/bundle/generate/genie_space.go @@ -62,21 +62,19 @@ type genieSpace struct { bind bool } -func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) string { +func (g *genieSpace) resolveFromID(ctx context.Context, b *bundle.Bundle) (string, error) { w := b.WorkspaceClient(ctx) obj, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ SpaceId: g.existingID, }) if err != nil { if apierr.IsMissing(err) { - logdiag.LogError(ctx, fmt.Errorf("genie space with ID %s not found", g.existingID)) - return "" + return "", fmt.Errorf("genie space with ID %s not found", g.existingID) } - logdiag.LogError(ctx, err) - return "" + return "", err } - return obj.SpaceId + return obj.SpaceId, nil } func (g *genieSpace) saveSerializedGenieSpace(ctx context.Context, b *bundle.Bundle, genieSpace *dashboards.GenieSpace, filename string) error { @@ -169,16 +167,14 @@ func (g *genieSpace) saveConfiguration(ctx context.Context, b *bundle.Bundle, ge return nil } -func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) { +func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle.Bundle) error { resource, ok := b.Config.Resources.GenieSpaces[g.resource] if !ok { - logdiag.LogError(ctx, fmt.Errorf("genie space resource %q is not defined", g.resource)) - return + return fmt.Errorf("genie space resource %q is not defined", g.resource) } if resource.FilePath == "" { - logdiag.LogError(ctx, fmt.Errorf("genie space resource %q has no file path defined", g.resource)) - return + return fmt.Errorf("genie space resource %q has no file path defined", g.resource) } genieSpaceID := resource.ID @@ -193,8 +189,7 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. IncludeSerializedSpace: true, }) if err != nil { - logdiag.LogError(ctx, err) - return + return err } // Genie has no remote modification timestamp we can poll. Compare @@ -205,27 +200,25 @@ func (g *genieSpace) updateGenieSpaceForResource(ctx context.Context, b *bundle. if !first { differs, err := genieSpaceBodyDiffersFromDisk(genieSpace.SerializedSpace, genieSpacePath) if err != nil { - logdiag.LogError(ctx, err) - return + return err } shouldSave = differs } if shouldSave { if err := g.saveSerializedGenieSpace(ctx, b, genieSpace, genieSpacePath); err != nil { - logdiag.LogError(ctx, err) - return + return err } } if !g.watch { - return + return nil } first = false select { case <-ctx.Done(): - return + return nil case <-time.After(genieSpaceWatchInterval): } } @@ -251,15 +244,14 @@ func genieSpaceBodyDiffersFromDisk(remoteSerialized, filename string) (bool, err return !bytes.Equal(canonical, onDisk), nil } -func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) { +func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, genieSpaceID string) error { w := b.WorkspaceClient(ctx) genieSpace, err := w.Genie.GetSpace(ctx, dashboards.GenieGetSpaceRequest{ SpaceId: genieSpaceID, IncludeSerializedSpace: true, }) if err != nil { - logdiag.LogError(ctx, err) - return + return err } key := g.cmd.Flag("key").Value.String() @@ -268,21 +260,21 @@ func (g *genieSpace) generateForExisting(ctx context.Context, b *bundle.Bundle, } err = g.saveConfiguration(ctx, b, genieSpace, key) if err != nil { - logdiag.LogError(ctx, err) - return + return err } if g.bind { err = deployment.BindResource(g.cmd, key, genieSpaceID, true, false, true) if err != nil { - logdiag.LogError(ctx, err) - return + return err } cmdio.LogString(ctx, fmt.Sprintf("Successfully bound genie space with an id '%s'", genieSpaceID)) } + + return nil } -func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { +func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) error { // Make the paths absolute if they aren't already. if !filepath.IsAbs(g.resourceDir) { g.resourceDir = filepath.Join(b.BundleRootPath, g.resourceDir) @@ -294,88 +286,84 @@ func (g *genieSpace) initialize(ctx context.Context, b *bundle.Bundle) { // Make sure we know how the genie space path is relative to the resource path. rel, err := filepath.Rel(g.resourceDir, g.genieSpaceDir) if err != nil { - logdiag.LogError(ctx, err) - return + return err } g.relativeGenieSpaceDir = filepath.ToSlash(rel) + return nil } -func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) { - phases.Initialize(ctx, b) - if logdiag.HasError(ctx) { - return +func (g *genieSpace) runForResource(ctx context.Context, b *bundle.Bundle) error { + if err := phases.Initialize(ctx, b); err != nil { + return err } requiredEngine, err := utils.ResolveEngineSetting(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return + return err } - ctx, stateDesc := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) - if logdiag.HasError(ctx) { - return + ctx, stateDesc, err := statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(true), requiredEngine) + if err != nil { + return err } var state statemgmt.ExportedResourcesMap if stateDesc.Engine.IsDirect() { _, localPath := b.StateFilenameDirect(ctx) if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { - logdiag.LogError(ctx, err) - return + return err } state = b.DeploymentBundle.ExportState(ctx) } else { - var err error state, err = terraform.ParseResourcesState(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return + return err } } - bundle.ApplySeqContext(ctx, b, + if err := bundle.ApplySeqContext(ctx, b, statemgmt.Load(state), - ) - if logdiag.HasError(ctx) { - return + ); err != nil { + return err } - g.updateGenieSpaceForResource(ctx, b) + return g.updateGenieSpaceForResource(ctx, b) } -func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) { +func (g *genieSpace) runForExisting(ctx context.Context, b *bundle.Bundle) error { // Resolve the ID of the genie space to generate configuration for. - genieSpaceID := g.resolveFromID(ctx, b) - if logdiag.HasError(ctx) { - return + genieSpaceID, err := g.resolveFromID(ctx, b) + if err != nil { + return err } - g.generateForExisting(ctx, b, genieSpaceID) + return g.generateForExisting(ctx, b, genieSpaceID) } func (g *genieSpace) RunE(cmd *cobra.Command, args []string) error { ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } - g.initialize(ctx, b) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := g.initialize(ctx, b); err != nil { + return root.RenderAndReturnError(ctx, err) } if g.resource != "" { - g.runForResource(ctx, b) + err = g.runForResource(ctx, b) } else { - g.runForExisting(ctx, b) + err = g.runForExisting(ctx, b) } - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err != nil { + return root.RenderAndReturnError(ctx, err) } return nil @@ -388,8 +376,8 @@ func filterGenieSpaces(ref resources.Reference) bool { // genieSpaceResourceCompletion executes to autocomplete the argument to the resource flag. func genieSpaceResourceCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/bundle/generate/genie_space_test.go b/cmd/bundle/generate/genie_space_test.go index 746b2b1c3af..3942c2e9aa9 100644 --- a/cmd/bundle/generate/genie_space_test.go +++ b/cmd/bundle/generate/genie_space_test.go @@ -67,7 +67,7 @@ func TestGenieSpace_UpdateForResource_WritesFileWhenNotWatching(t *testing.T) { ctx, _ := cmdio.NewTestContextWithStdout(t.Context()) ctx = logdiag.InitContext(ctx) logdiag.SetCollect(ctx, true) - g.updateGenieSpaceForResource(ctx, b) + require.NoError(t, g.updateGenieSpaceForResource(ctx, b)) require.Empty(t, logdiag.FlushCollected(ctx)) @@ -107,7 +107,9 @@ func TestGenieSpace_UpdateForResource_WatchExitsOnCancel(t *testing.T) { done := make(chan struct{}) go func() { - g.updateGenieSpaceForResource(ctx, b) + // Returns nil once the context is cancelled below; the test asserts the + // initial save landed and that this goroutine exits promptly. + _ = g.updateGenieSpaceForResource(ctx, b) close(done) }() diff --git a/cmd/bundle/generate/job.go b/cmd/bundle/generate/job.go index c3aba49c5f2..171e96482be 100644 --- a/cmd/bundle/generate/job.go +++ b/cmd/bundle/generate/job.go @@ -69,8 +69,11 @@ After generation, you can deploy this job to other targets using: ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } diff --git a/cmd/bundle/generate/pipeline.go b/cmd/bundle/generate/pipeline.go index 35fb073cadd..8116029c6d6 100644 --- a/cmd/bundle/generate/pipeline.go +++ b/cmd/bundle/generate/pipeline.go @@ -68,8 +68,11 @@ like catalogs, schemas, and compute configurations per target.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if b == nil || logdiag.HasError(ctx) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { + return root.RenderAndReturnError(ctx, err) + } + if b == nil { return root.ErrAlreadyPrinted } diff --git a/cmd/bundle/open.go b/cmd/bundle/open.go index d357b4f39e1..08e12469ef9 100644 --- a/cmd/bundle/open.go +++ b/cmd/bundle/open.go @@ -106,8 +106,8 @@ Use after deployment to quickly navigate to your resources in the workspace.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/bundle/plan.go b/cmd/bundle/plan.go index 20df8cb5f0f..0c9c7f752e6 100644 --- a/cmd/bundle/plan.go +++ b/cmd/bundle/plan.go @@ -11,7 +11,6 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/flags" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -61,9 +60,9 @@ It is useful for previewing changes before running 'bundle deploy'.`, } ctx := cmd.Context() - plan := phases.RunPlan(ctx, b, stateDesc.Engine) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + plan, err := phases.RunPlan(ctx, b, stateDesc.Engine) + if err != nil { + return root.RenderAndReturnError(ctx, err) } // Count actions by type and collect formatted actions @@ -114,16 +113,9 @@ It is useful for previewing changes before running 'bundle deploy'.`, return err } fmt.Fprintln(out, string(buf)) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted - } return nil } - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted - } - return nil } diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index e98fe59ac4e..6eee15f5494 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -227,8 +227,8 @@ Example usage: ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/bundle/summary.go b/cmd/bundle/summary.go index b3a55a607cc..2d08997bfda 100644 --- a/cmd/bundle/summary.go +++ b/cmd/bundle/summary.go @@ -6,7 +6,6 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/flags" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -35,16 +34,7 @@ Useful after deployment to see what was created and where to find it.`, if err != nil { return err } - err = showSummary(cmd, b) - if err != nil { - return err - } - - if logdiag.HasError(cmd.Context()) { - return root.ErrAlreadyPrinted - } - - return nil + return showSummary(cmd, b) } return cmd diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index d61c4525530..1a0fb0c55cc 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -19,7 +19,6 @@ import ( "github.com/databricks/cli/bundle/statemgmt" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/internal/build" - "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/dyn" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/logdiag" @@ -99,7 +98,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // Load bundle config and apply target - b = root.MustConfigureBundle(cmd) + b, cfgErr := root.MustConfigureBundle(cmd) // Log deploy telemetry on all exit paths. This is a defer to ensure // telemetry is logged even when the deploy command fails, for both @@ -109,6 +108,9 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if b == nil { return } + // Prefer the first logged error summary: when a mutator renders its + // errors and returns the opaque ErrAlreadyPrinted sentinel, retErr no + // longer carries the original message. errMsg := logdiag.GetFirstErrorSummary(ctx) if errMsg == "" && retErr != nil && !errors.Is(retErr, root.ErrAlreadyPrinted) { errMsg = retErr.Error() @@ -117,50 +119,53 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle }() } - if logdiag.HasError(ctx) { - return b, nil, root.ErrAlreadyPrinted + if cfgErr != nil { + return b, nil, root.RenderAndReturnError(ctx, cfgErr) } variables, err := cmd.Flags().GetStringSlice("var") if err != nil { - logdiag.LogDiag(ctx, diag.FromErr(err)[0]) return b, nil, err } // Initialize variables by assigning them values passed as command line flags - configureVariables(cmd, b, variables) + if err := configureVariables(cmd, b, variables); err != nil { + return b, nil, root.RenderAndReturnError(ctx, err) + } - if b == nil || logdiag.HasError(ctx) { + if b == nil { return b, nil, root.ErrAlreadyPrinted } ctx = cmd.Context() if opts.InitFunc != nil { - bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) { opts.InitFunc(b) }) + if err := bundle.ApplyFuncContext(ctx, b, func(context.Context, *bundle.Bundle) { opts.InitFunc(b) }); err != nil { + return b, nil, root.RenderAndReturnError(ctx, err) + } } + var initErr error if !opts.SkipInitialize { t0 := time.Now() - phases.Initialize(ctx, b) + initErr = phases.Initialize(ctx, b) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Initialize", Value: time.Since(t0).Milliseconds(), }) - // not checking error right away here, add locations first + // not returning the error right away here, add locations first } if b != nil { // Include location information in the output if the flag is set. if opts.IncludeLocations { - bundle.ApplyContext(ctx, b, mutator.PopulateLocations()) - if logdiag.HasError(ctx) { - return b, nil, root.ErrAlreadyPrinted + if err := bundle.ApplyContext(ctx, b, mutator.PopulateLocations()); err != nil { + return b, nil, root.RenderAndReturnError(ctx, err) } } } - if logdiag.HasError(ctx) { - return b, nil, root.ErrAlreadyPrinted + if initErr != nil { + return b, nil, root.RenderAndReturnError(ctx, initErr) } if opts.PostInitFunc != nil { @@ -179,9 +184,10 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } // PullResourcesState depends on stateFiler which needs b.Config.Workspace.StatePath which is set in phases.Initialize - ctx, stateDesc = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(opts.AlwaysPull), requiredEngine) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + var pullErr error + ctx, stateDesc, pullErr = statemgmt.PullResourcesState(ctx, b, statemgmt.AlwaysPull(opts.AlwaysPull), requiredEngine) + if pullErr != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, pullErr) } cmd.SetContext(ctx) @@ -190,8 +196,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // The engine is only known for certain after the state is pulled, so reject it // here rather than silently planning/deploying every resource on terraform. if len(b.Select) > 0 && !stateDesc.Engine.IsDirect() { - logdiag.LogError(ctx, errors.New("--select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct")) - return b, stateDesc, root.ErrAlreadyPrinted + return b, stateDesc, root.RenderAndReturnError(ctx, errors.New("--select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct")) } // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) @@ -199,8 +204,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if needDirectState { _, localPath := b.StateFilenameDirect(ctx) if err := b.DeploymentBundle.StateDB.Open(ctx, localPath, dstate.WithRecovery(true), dstate.WithWrite(false)); err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } @@ -217,8 +221,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle var err error state, err = terraform.ParseResourcesState(ctx, b) if err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } mutators := []bundle.Mutator{ @@ -228,9 +231,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.InitIDs { mutators = append(mutators, mutator.InitializeURLs()) } - bundle.ApplySeqContext(ctx, b, mutators...) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if err := bundle.ApplySeqContext(ctx, b, mutators...); err != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } } @@ -239,8 +241,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.ReadPlanPath != "" { if !stateDesc.Engine.IsDirect() { - logdiag.LogError(ctx, errors.New("--plan is only supported with direct engine (set bundle.engine to \"direct\" or DATABRICKS_BUNDLE_ENGINE=direct)")) - return b, stateDesc, root.ErrAlreadyPrinted + return b, stateDesc, root.RenderAndReturnError(ctx, errors.New("--plan is only supported with direct engine (set bundle.engine to \"direct\" or DATABRICKS_BUNDLE_ENGINE=direct)")) } opts.Build = false opts.PreDeployChecks = false @@ -248,8 +249,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle var err error plan, err = deployplan.LoadPlanFromFile(opts.ReadPlanPath) if err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted + return b, stateDesc, root.RenderAndReturnError(ctx, err) } currentVersion := build.GetInfo().Version if plan.CLIVersion != currentVersion { @@ -260,8 +260,7 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // This must happen before any file operations err = direct.ValidatePlanAgainstState(&b.DeploymentBundle.StateDB, plan) if err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } else if opts.Deploy { opts.Build = true @@ -270,29 +269,27 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.FastValidate { t1 := time.Now() - bundle.ApplyContext(ctx, b, validate.FastValidate()) + fastValidateErr := bundle.ApplyContext(ctx, b, validate.FastValidate()) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "validate.FastValidate", Value: time.Since(t1).Milliseconds(), }) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if fastValidateErr != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, fastValidateErr) } // Pipeline CLI only validation. if opts.IsPipelinesCLI { - rejectDefinitions(ctx, b) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if err := rejectDefinitions(ctx, b); err != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } } if opts.Validate { - validate.Validate(ctx, b) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if err := validate.Validate(ctx, b); err != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } @@ -300,23 +297,22 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.Build { t2 := time.Now() - libs = phases.Build(ctx, b) + var buildErr error + libs, buildErr = phases.Build(ctx, b) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Build", Value: time.Since(t2).Milliseconds(), }) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if buildErr != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, buildErr) } } if opts.PreDeployChecks { downgradeWarningToError := !opts.Deploy - phases.PreDeployChecks(ctx, b, downgradeWarningToError, stateDesc.Engine) - - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if err := phases.PreDeployChecks(ctx, b, downgradeWarningToError, stateDesc.Engine); err != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } @@ -329,21 +325,19 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle } t3 := time.Now() - phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, libs, plan) + deployErr := phases.Deploy(ctx, b, outputHandler, stateDesc.Engine, libs, plan) b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{ Key: "phases.Deploy", Value: time.Since(t3).Milliseconds(), }) - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if deployErr != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, deployErr) } if b != nil && stateDesc != nil && stateDesc.Engine.IsDirect() && stateDesc.HasRemoteTerraformState() { - statemgmt.BackupRemoteTerraformState(ctx, b) - - if logdiag.HasError(ctx) { - return b, stateDesc, root.ErrAlreadyPrinted + if err := statemgmt.BackupRemoteTerraformState(ctx, b); err != nil { + return b, stateDesc, root.RenderAndReturnError(ctx, err) } } } @@ -383,7 +377,7 @@ func ResolveEngineSetting(ctx context.Context, b *bundle.Bundle) (engine.EngineS return engine.EngineSetting{}, nil } -func rejectDefinitions(ctx context.Context, b *bundle.Bundle) { +func rejectDefinitions(ctx context.Context, b *bundle.Bundle) error { if b.Config.Definitions != nil { v := dyn.GetValue(b.Config.Value(), "definitions") loc := v.Locations() @@ -391,8 +385,9 @@ func rejectDefinitions(ctx context.Context, b *bundle.Bundle) { if len(loc) > 0 { filename = filepath.ToSlash(loc[0].File) } - logdiag.LogError(ctx, errors.New(filename+` seems to be formatted for open-source Spark Declarative Pipelines. + return errors.New(filename + ` seems to be formatted for open-source Spark Declarative Pipelines. Pipelines CLI currently only supports Lakeflow Spark Declarative Pipelines development. -To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init".`)) +To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init".`) } + return nil } diff --git a/cmd/bundle/utils/utils.go b/cmd/bundle/utils/utils.go index 3c4bd1a5b98..8750005c59e 100644 --- a/cmd/bundle/utils/utils.go +++ b/cmd/bundle/utils/utils.go @@ -4,15 +4,15 @@ import ( "context" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) -func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) { - bundle.ApplyFuncContext(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) { - err := b.Config.InitializeVariables(variables) - if err != nil { - logdiag.LogError(ctx, err) - } - }) +func configureVariables(cmd *cobra.Command, b *bundle.Bundle, variables []string) error { + var initErr error + if err := bundle.ApplyFuncContext(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) { + initErr = b.Config.InitializeVariables(variables) + }); err != nil { + return err + } + return initErr } diff --git a/cmd/bundle/validate.go b/cmd/bundle/validate.go index a2ec31f721b..56627b57127 100644 --- a/cmd/bundle/validate.go +++ b/cmd/bundle/validate.go @@ -2,6 +2,7 @@ package bundle import ( "encoding/json" + "errors" "fmt" "github.com/databricks/cli/bundle" @@ -57,9 +58,8 @@ Please run this command before deploying to ensure configuration quality.`, }) ctx := cmd.Context() - if err != nil && err != root.ErrAlreadyPrinted { - logdiag.LogError(ctx, err) - err = root.ErrAlreadyPrinted + if err != nil && !errors.Is(err, root.ErrAlreadyPrinted) { + err = root.RenderAndReturnError(ctx, err) } // output before checking the error on purpose diff --git a/cmd/labs/project/entrypoint.go b/cmd/labs/project/entrypoint.go index 1ffb4a8aaf8..cd6929ecad2 100644 --- a/cmd/labs/project/entrypoint.go +++ b/cmd/labs/project/entrypoint.go @@ -202,7 +202,10 @@ func (e *Entrypoint) getLoginConfig(cmd *cobra.Command) (*loginConfig, *config.C } if e.IsBundleAware { ctx := cmd.Context() - b := root.TryConfigureBundle(cmd) + b, err := root.TryConfigureBundle(cmd) + if err != nil { + return nil, nil, root.RenderAndReturnError(ctx, err) + } if b != nil { log.Infof(ctx, "Using login configuration from Databricks Asset Bundle") return &loginConfig{}, b.WorkspaceClient(ctx).Config, nil diff --git a/cmd/pipelines/deploy.go b/cmd/pipelines/deploy.go index 2c8d27de140..556b5f45815 100644 --- a/cmd/pipelines/deploy.go +++ b/cmd/pipelines/deploy.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -58,9 +57,8 @@ func deployCommand() *cobra.Command { } ctx := cmd.Context() - bundle.ApplyContext(ctx, b, mutator.InitializeURLs()) - if logdiag.HasError(ctx) { - return root.ErrAlreadyPrinted + if err := bundle.ApplyContext(ctx, b, mutator.InitializeURLs()); err != nil { + return root.RenderAndReturnError(ctx, err) } for _, group := range b.Config.Resources.AllResources() { diff --git a/cmd/pipelines/dry_run.go b/cmd/pipelines/dry_run.go index ec74f7f7b0f..2558ecdefa7 100644 --- a/cmd/pipelines/dry_run.go +++ b/cmd/pipelines/dry_run.go @@ -103,8 +103,8 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w } cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/pipelines/open.go b/cmd/pipelines/open.go index 6a8419703be..531be381213 100644 --- a/cmd/pipelines/open.go +++ b/cmd/pipelines/open.go @@ -89,8 +89,8 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/pipelines/run.go b/cmd/pipelines/run.go index 9f81961b6f0..fca2c1088a9 100644 --- a/cmd/pipelines/run.go +++ b/cmd/pipelines/run.go @@ -345,8 +345,8 @@ Refreshes all tables in the pipeline unless otherwise specified.`, ctx := logdiag.InitContext(cmd.Context()) cmd.SetContext(ctx) - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/pipelines/stop.go b/cmd/pipelines/stop.go index 40e4b5e4660..86fbd7a5bbc 100644 --- a/cmd/pipelines/stop.go +++ b/cmd/pipelines/stop.go @@ -14,7 +14,6 @@ import ( "github.com/databricks/cli/cmd/bundle/utils" "github.com/databricks/cli/cmd/root" "github.com/databricks/cli/libs/cmdio" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -86,8 +85,8 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w // TODO: This autocomplete functionality was copied from cmd/bundle/run.go and is not working properly. cmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - b := root.MustConfigureBundle(cmd) - if logdiag.HasError(cmd.Context()) { + b, err := root.MustConfigureBundle(cmd) + if err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/root/auth.go b/cmd/root/auth.go index 84c09e8a7ec..f8e0fd8fe7a 100644 --- a/cmd/root/auth.go +++ b/cmd/root/auth.go @@ -340,11 +340,11 @@ func MustWorkspaceClient(cmd *cobra.Command, args []string) error { // Try to load a bundle configuration if we're allowed to by the caller (see `./auth_options.go`). if !shouldSkipLoadBundle(cmd.Context()) { - b := TryConfigureBundle(cmd) + b, err := TryConfigureBundle(cmd) // Use the updated context from the command after TryConfigureBundle ctx = cmd.Context() - if logdiag.HasError(ctx) { - return ErrAlreadyPrinted + if err != nil { + return RenderAndReturnError(ctx, err) } if b != nil { diff --git a/cmd/root/bundle.go b/cmd/root/bundle.go index a17d88f5fcf..aa39ca7d823 100644 --- a/cmd/root/bundle.go +++ b/cmd/root/bundle.go @@ -16,7 +16,6 @@ import ( "github.com/databricks/cli/libs/databrickscfg" "github.com/databricks/cli/libs/databrickscfg/profile" envlib "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/logdiag" "github.com/spf13/cobra" ) @@ -68,7 +67,7 @@ func getProfile(cmd *cobra.Command) (value string) { } // configureProfile applies the profile flag to the bundle. -func configureProfile(cmd *cobra.Command, b *bundle.Bundle) { +func configureProfile(cmd *cobra.Command, b *bundle.Bundle) error { profile := getProfile(cmd) // Fall back to [__settings__].default_profile only when the bundle does @@ -83,10 +82,10 @@ func configureProfile(cmd *cobra.Command, b *bundle.Bundle) { } if profile == "" { - return + return nil } - bundle.ApplyFuncContext(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) { + return bundle.ApplyFuncContext(cmd.Context(), b, func(ctx context.Context, b *bundle.Bundle) { b.Config.Workspace.Profile = profile }) } @@ -153,21 +152,24 @@ func resolveProfileAmbiguity(cmd *cobra.Command, b *bundle.Bundle, originalErr e } // configureBundle loads the bundle configuration and configures flag values, if any. -func configureBundle(cmd *cobra.Command, b *bundle.Bundle) { +func configureBundle(cmd *cobra.Command, b *bundle.Bundle) error { // Load bundle and select target. ctx := cmd.Context() + var loadErr error if target := getTarget(cmd); target == "" { - phases.LoadDefaultTarget(ctx, b) + loadErr = phases.LoadDefaultTarget(ctx, b) } else { - phases.LoadNamedTarget(ctx, b, target) + loadErr = phases.LoadNamedTarget(ctx, b, target) } - if logdiag.HasError(ctx) { - return + if loadErr != nil { + return loadErr } // Configure the workspace profile if the flag has been set. - configureProfile(cmd, b) + if err := configureProfile(cmd, b); err != nil { + return err + } // Set the auth configuration in the command context. This can be used // downstream to initialize a API client. @@ -178,77 +180,84 @@ func configureBundle(cmd *cobra.Command, b *bundle.Bundle) { if err != nil { names, isMulti := databrickscfg.AsMultipleProfiles(err) if !isMulti { - logdiag.LogError(ctx, err) - return + return err } selected, resolveErr := resolveProfileAmbiguity(cmd, b, err, names) if resolveErr != nil { - logdiag.LogError(ctx, resolveErr) - return + return resolveErr } b.Config.Workspace.Profile = selected b.ClearWorkspaceClient(ctx) client, err = b.WorkspaceClientE(ctx) if err != nil { - logdiag.LogError(ctx, err) - return + return err } } ctx = cmdctx.SetConfigUsed(ctx, client.Config) cmd.SetContext(ctx) + return nil } // MustConfigureBundle configures a bundle on the command context. -func MustConfigureBundle(cmd *cobra.Command) *bundle.Bundle { +func MustConfigureBundle(cmd *cobra.Command) (*bundle.Bundle, error) { // A bundle may be configured on the context when testing. // If it is, return it immediately. b := bundle.GetOrNil(cmd.Context()) if b != nil { - return b + return b, nil } - b = bundle.MustLoad(cmd.Context()) + b, err := bundle.MustLoad(cmd.Context()) + if err != nil { + return b, err + } if b != nil { - configureBundle(cmd, b) + if err := configureBundle(cmd, b); err != nil { + return b, err + } } - return b + return b, nil } // TryConfigureBundle configures a bundle on the command context // if there is one, but doesn't fail if there isn't one. -func TryConfigureBundle(cmd *cobra.Command) *bundle.Bundle { +func TryConfigureBundle(cmd *cobra.Command) (*bundle.Bundle, error) { // A bundle may be configured on the context when testing. // If it is, return it immediately. b := bundle.GetOrNil(cmd.Context()) if b != nil { - return b + return b, nil } ctx := cmd.Context() - b = bundle.TryLoad(ctx) + b, err := bundle.TryLoad(ctx) + if err != nil { + return nil, err + } // No bundle is fine in this case. - if b == nil || logdiag.HasError(ctx) { - return nil + if b == nil { + return nil, nil } - configureBundle(cmd, b) - return b + if err := configureBundle(cmd, b); err != nil { + return b, err + } + return b, nil } // targetCompletion executes to autocomplete the argument to the target flag. func targetCompletion(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { ctx := cmd.Context() - b := bundle.MustLoad(ctx) - if b == nil || logdiag.HasError(ctx) { + b, err := bundle.MustLoad(ctx) + if err != nil || b == nil { return nil, cobra.ShellCompDirectiveError } // Load bundle but don't select a target (we're completing those). - phases.Load(ctx, b) - if logdiag.HasError(ctx) { + if err := phases.Load(ctx, b); err != nil { return nil, cobra.ShellCompDirectiveError } diff --git a/cmd/root/bundle_test.go b/cmd/root/bundle_test.go index 6116003ea70..3a49131d3e1 100644 --- a/cmd/root/bundle_test.go +++ b/cmd/root/bundle_test.go @@ -59,8 +59,12 @@ workspace: ctx := logdiag.InitContext(cmd.Context()) logdiag.SetCollect(ctx, true) cmd.SetContext(ctx) - _ = MustConfigureBundle(cmd) - return logdiag.FlushCollected(ctx) + _, cfgErr := MustConfigureBundle(cmd) + diags := logdiag.FlushCollected(ctx) + if cfgErr != nil { + diags = append(diags, diag.DiagnosticFromError(cfgErr)) + } + return diags } // setupBundleNameOnly writes a databricks.yml that declares only the bundle @@ -79,8 +83,12 @@ func setupBundleNameOnly(t *testing.T, cmd *cobra.Command) []diag.Diagnostic { ctx := logdiag.InitContext(cmd.Context()) logdiag.SetCollect(ctx, true) cmd.SetContext(ctx) - _ = MustConfigureBundle(cmd) - return logdiag.FlushCollected(ctx) + _, cfgErr := MustConfigureBundle(cmd) + diags := logdiag.FlushCollected(ctx) + if cfgErr != nil { + diags = append(diags, diag.DiagnosticFromError(cfgErr)) + } + return diags } func setupWithProfile(t *testing.T, cmd *cobra.Command, profile string) []diag.Diagnostic { @@ -99,8 +107,12 @@ workspace: ctx := logdiag.InitContext(cmd.Context()) logdiag.SetCollect(ctx, true) cmd.SetContext(ctx) - _ = MustConfigureBundle(cmd) - return logdiag.FlushCollected(ctx) + _, cfgErr := MustConfigureBundle(cmd) + diags := logdiag.FlushCollected(ctx) + if cfgErr != nil { + diags = append(diags, diag.DiagnosticFromError(cfgErr)) + } + return diags } func TestBundleConfigureDefault(t *testing.T) { @@ -315,8 +327,9 @@ func TestBundleConfigureWithDefaultProfile_BundleHostWins(t *testing.T) { ctx := logdiag.InitContext(cmd.Context()) logdiag.SetCollect(ctx, true) cmd.SetContext(ctx) - _ = MustConfigureBundle(cmd) + _, cfgErr := MustConfigureBundle(cmd) diags := logdiag.FlushCollected(ctx) + require.NoError(t, cfgErr) require.Empty(t, diags) assert.Equal(t, "https://b.test", cmdctx.ConfigUsed(cmd.Context()).Host) assert.Equal(t, "PROFILE-2", cmdctx.ConfigUsed(cmd.Context()).Profile) @@ -369,7 +382,7 @@ workspace: ctx := logdiag.InitContext(cmd.Context()) logdiag.SetCollect(ctx, true) cmd.SetContext(ctx) - _ = MustConfigureBundle(cmd) + _, _ = MustConfigureBundle(cmd) }() // Verify the prompt fires by reading output from stderr. diff --git a/cmd/root/report_error.go b/cmd/root/report_error.go new file mode 100644 index 00000000000..0afdd1fd3c5 --- /dev/null +++ b/cmd/root/report_error.go @@ -0,0 +1,15 @@ +package root + +import ( + "context" + + "github.com/databricks/cli/libs/logdiag" +) + +// RenderAndReturnError renders err to the user as one or more diagnostics and +// returns it wrapped as already-printed, so the top-level command does not print +// it again. It is the idempotent boundary fallback for any error not already +// flushed at its source; see logdiag.FlushError. +func RenderAndReturnError(ctx context.Context, err error) error { + return logdiag.FlushError(ctx, err) +} diff --git a/cmd/root/silent_err.go b/cmd/root/silent_err.go index b361cc6b404..f76c1f5dc60 100644 --- a/cmd/root/silent_err.go +++ b/cmd/root/silent_err.go @@ -1,7 +1,11 @@ package root -import "errors" +import "github.com/databricks/cli/libs/logdiag" // ErrAlreadyPrinted is not printed to the user. It's used to signal that the command should exit with an error, // but the error message was already printed. -var ErrAlreadyPrinted = errors.New("AlreadyPrinted") +// +// It aliases logdiag.ErrAlreadyPrinted so that errors flushed deep in the bundle +// pipeline (which cannot import this package) are recognized as already-printed +// here via errors.Is. +var ErrAlreadyPrinted = logdiag.ErrAlreadyPrinted diff --git a/libs/diag/diagnostic.go b/libs/diag/diagnostic.go index fe7090462d4..c266d9e9164 100644 --- a/libs/diag/diagnostic.go +++ b/libs/diag/diagnostic.go @@ -30,13 +30,29 @@ type Diagnostic struct { ID ID } +// Error implements the error interface so an error-severity Diagnostic can be +// returned and propagated as a regular Go error. The message mirrors the +// formatting used by [Diagnostics.Error]. +func (d Diagnostic) Error() string { + message := d.Detail + if message == "" { + message = d.Summary + } + if d.ID != "" { + message = string(d.ID) + ": " + message + } + return message +} + // Errorf creates a new error diagnostic. -func Errorf(format string, args ...any) Diagnostics { - return []Diagnostic{ - { - Severity: Error, - Summary: fmt.Sprintf(format, args...), - }, +// +// The returned value implements the error interface so it can be returned and +// propagated like any other Go error while still carrying the diagnostic's +// Summary/Detail/ID for rendering at the top level. +func Errorf(format string, args ...any) error { + return Diagnostic{ + Severity: Error, + Summary: fmt.Sprintf(format, args...), } } @@ -54,6 +70,21 @@ func FromErr(err error) Diagnostics { } } +// DiagnosticFromError converts an error into a single error-severity Diagnostic +// for rendering. If the error (or anything it wraps) is already a Diagnostic, it +// is returned unchanged so its Locations/Paths/Detail/ID render as authored. +// Otherwise the error is formatted as an API/error diagnostic. +func DiagnosticFromError(err error) Diagnostic { + if d, ok := errors.AsType[Diagnostic](err); ok { + return d + } + return Diagnostic{ + Severity: Error, + Summary: FormatAPIErrorSummary(err), + Detail: FormatAPIErrorDetails(err), + } +} + // FromErr returns a new warning diagnostic from the specified error, if any. func WarningFromErr(err error) Diagnostics { if err == nil { @@ -110,21 +141,27 @@ func (ds Diagnostics) HasError() bool { return false } -// Return first error in the set of diagnostics. +// Error returns the error-severity diagnostics in the set as a single error, or +// nil if there are none. A single error is returned as the [Diagnostic] itself +// (it implements the error interface); multiple errors are combined with +// [errors.Join] so they unpack via Unwrap() []error and render as separate +// diagnostic blocks (see [FlushError]). Either way Locations/Paths/Detail/ID are +// preserved and render correctly when surfaced via [DiagnosticFromError]. func (ds Diagnostics) Error() error { + var errs []error for _, d := range ds { if d.Severity == Error { - message := d.Detail - if message == "" { - message = d.Summary - } - if d.ID != "" { - message = string(d.ID) + ": " + message - } - return errors.New(message) + errs = append(errs, d) } } - return nil + switch len(errs) { + case 0: + return nil + case 1: + return errs[0] + default: + return errors.Join(errs...) + } } // Filter returns a new list of diagnostics that match the specified severity. diff --git a/libs/dyn/convert/normalize.go b/libs/dyn/convert/normalize.go index 79cfee37441..f92530ce4e2 100644 --- a/libs/dyn/convert/normalize.go +++ b/libs/dyn/convert/normalize.go @@ -66,7 +66,10 @@ func (n normalizeOptions) normalizeType(typ reflect.Type, src dyn.Value, seen [] // Fall through to the error case. } - return dyn.InvalidValue, diag.Errorf("unsupported type: %s", typ.Kind()) + return dyn.InvalidValue, diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("unsupported type: %s", typ.Kind()), + }} } func nullWarning(expected dyn.Kind, src dyn.Value, path dyn.Path) diag.Diagnostic { @@ -450,7 +453,10 @@ func (n normalizeOptions) normalizeInterface(_ reflect.Type, src dyn.Value, path case dyn.KindNil: // Fall through default: - return dyn.InvalidValue, diag.Errorf("unsupported kind: %s", src.Kind()) + return dyn.InvalidValue, diag.Diagnostics{{ + Severity: diag.Error, + Summary: fmt.Sprintf("unsupported kind: %s", src.Kind()), + }} } return src, nil diff --git a/libs/logdiag/flush_error.go b/libs/logdiag/flush_error.go new file mode 100644 index 00000000000..9f7fd522d27 --- /dev/null +++ b/libs/logdiag/flush_error.go @@ -0,0 +1,87 @@ +package logdiag + +import ( + "context" + "errors" + + "github.com/databricks/cli/libs/diag" +) + +// ErrAlreadyPrinted marks an error whose message has already been rendered to +// the user. The top-level command uses it (via errors.Is) to avoid printing the +// error a second time. +var ErrAlreadyPrinted = errors.New("AlreadyPrinted") + +// alreadyPrinted wraps an error that FlushError has already rendered. It matches +// ErrAlreadyPrinted via errors.Is while preserving the underlying error for +// inspection (telemetry, errors.As). +type alreadyPrinted struct{ err error } + +func (e alreadyPrinted) Error() string { return e.err.Error() } + +func (e alreadyPrinted) Unwrap() error { return e.err } + +func (e alreadyPrinted) Is(target error) bool { return target == ErrAlreadyPrinted } + +// FlushError renders err to the user immediately as one or more diagnostics and +// returns it wrapped so it matches ErrAlreadyPrinted via errors.Is. Rendering at +// the point of failure (rather than only at the top-level boundary) ensures the +// user sees the error before any slow deferred cleanup runs (lock release, WAL +// finalize, remote-state backup), which would otherwise delay or, on an +// interrupted process, hide it. +// +// FlushError is idempotent: an already-printed error is returned unchanged +// without re-rendering, so callers can flush liberally and upstream boundaries +// can flush again as a fallback without double-printing. It returns nil for nil. +// +// An errors.Join tree is expanded so each leaf error renders as its own +// diagnostic block, matching the previous diagnostics-based pipeline where +// parallel mutators and resources each reported separately. +func FlushError(ctx context.Context, err error) error { + if err == nil { + return nil + } + if errors.Is(err, ErrAlreadyPrinted) { + return err + } + for _, e := range flattenErrors(err) { + LogDiag(ctx, diag.DiagnosticFromError(e)) + } + return alreadyPrinted{err} +} + +// Flush logs every diagnostic in ds (warnings, recommendations and errors) and +// returns ErrAlreadyPrinted if any of them is an error, or nil otherwise. +// +// It is the convenience form of the "render diagnostics, then signal failure" +// pattern used by mutators that report more than one error: accumulate them +// into a diag.Diagnostics and `return logdiag.Flush(ctx, diags)`. +func Flush(ctx context.Context, ds diag.Diagnostics) error { + hasError := false + for _, d := range ds { + LogDiag(ctx, d) + if d.Severity == diag.Error { + hasError = true + } + } + if hasError { + return ErrAlreadyPrinted + } + return nil +} + +func flattenErrors(err error) []error { + if err == nil { + return nil + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + var out []error + for _, e := range joined.Unwrap() { + out = append(out, flattenErrors(e)...) + } + if len(out) > 0 { + return out + } + } + return []error{err} +} diff --git a/libs/logdiag/flush_error_test.go b/libs/logdiag/flush_error_test.go new file mode 100644 index 00000000000..46ac64b5b43 --- /dev/null +++ b/libs/logdiag/flush_error_test.go @@ -0,0 +1,49 @@ +package logdiag + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFlushErrorIdempotent(t *testing.T) { + ctx := InitContext(t.Context()) + SetCollect(ctx, true) + + flushed := FlushError(ctx, errors.New("boom")) + require.Error(t, flushed) + // The wrapped error matches the sentinel but preserves the message. + assert.ErrorIs(t, flushed, ErrAlreadyPrinted) + assert.Equal(t, "boom", flushed.Error()) + + // Flushing the already-flushed error is a no-op: it returns the same error + // and does not render a second diagnostic. + again := FlushError(ctx, flushed) + assert.Equal(t, flushed, again) + + diags := FlushCollected(ctx) + assert.Len(t, diags, 1) + assert.Equal(t, "boom", diags[0].Summary) +} + +func TestFlushErrorNil(t *testing.T) { + ctx := InitContext(t.Context()) + SetCollect(ctx, true) + + assert.NoError(t, FlushError(ctx, nil)) + assert.Empty(t, FlushCollected(ctx)) +} + +func TestFlushErrorJoinRendersEachLeaf(t *testing.T) { + ctx := InitContext(t.Context()) + SetCollect(ctx, true) + + _ = FlushError(ctx, errors.Join(errors.New("one"), errors.New("two"))) + + diags := FlushCollected(ctx) + require.Len(t, diags, 2) + assert.Equal(t, "one", diags[0].Summary) + assert.Equal(t, "two", diags[1].Summary) +} diff --git a/libs/logdiag/logdiag.go b/libs/logdiag/logdiag.go index 28ed3b5ba21..36441acf8f7 100644 --- a/libs/logdiag/logdiag.go +++ b/libs/logdiag/logdiag.go @@ -31,7 +31,9 @@ type LogDiagData struct { Collect bool Collected []diag.Diagnostic - // Summary of the first error diagnostic logged, if any. + // Summary of the first error diagnostic logged, if any. Used for deploy + // telemetry, where the returned error may be the opaque ErrAlreadyPrinted + // sentinel and no longer carries the original message. FirstErrorSummary string } @@ -76,20 +78,22 @@ func Copy(ctx context.Context) LogDiagData { return *val } -func HasError(ctx context.Context) bool { +func NumWarnings(ctx context.Context) int { val := read(ctx) val.mu.Lock() defer val.mu.Unlock() - return val.Errors > 0 + return val.Warnings } -func NumWarnings(ctx context.Context) int { +// GetFirstErrorSummary returns the summary of the first error diagnostic logged, +// or an empty string if no errors have been logged. +func GetFirstErrorSummary(ctx context.Context) string { val := read(ctx) val.mu.Lock() defer val.mu.Unlock() - return val.Warnings + return val.FirstErrorSummary } func SetSeverity(ctx context.Context, target diag.Severity) { @@ -126,16 +130,6 @@ func FlushCollected(ctx context.Context) diag.Diagnostics { return result } -// GetFirstErrorSummary returns the summary of the first error diagnostic -// logged, or an empty string if no errors have been logged. -func GetFirstErrorSummary(ctx context.Context) string { - val := read(ctx) - val.mu.Lock() - defer val.mu.Unlock() - - return val.FirstErrorSummary -} - func LogDiag(ctx context.Context, d diag.Diagnostic) { val := read(ctx) val.mu.Lock() diff --git a/libs/logdiag/logdiag_test.go b/libs/logdiag/logdiag_test.go index ecf12ee78d1..feac18ef7b5 100644 --- a/libs/logdiag/logdiag_test.go +++ b/libs/logdiag/logdiag_test.go @@ -1,9 +1,9 @@ package logdiag_test import ( - "errors" "testing" + "github.com/databricks/cli/libs/diag" "github.com/databricks/cli/libs/logdiag" "github.com/stretchr/testify/assert" ) @@ -14,11 +14,9 @@ func TestIsolatedContext(t *testing.T) { isolated := logdiag.IsolatedContext(ctx) logdiag.SetCollect(isolated, true) - logdiag.LogError(isolated, errors.New("inner failure")) + logdiag.LogDiag(isolated, diag.Diagnostic{Severity: diag.Error, Summary: "inner failure"}) - assert.True(t, logdiag.HasError(isolated)) + // The error is recorded in the isolated context only, not the parent. assert.Len(t, logdiag.FlushCollected(isolated), 1) - - assert.False(t, logdiag.HasError(ctx)) assert.Empty(t, logdiag.FlushCollected(ctx)) } diff --git a/libs/template/renderer_test.go b/libs/template/renderer_test.go index bb839628627..783e954c7ad 100644 --- a/libs/template/renderer_test.go +++ b/libs/template/renderer_test.go @@ -68,28 +68,31 @@ func assertBuiltinTemplateValid(t *testing.T, template string, settings map[stri ctx = logdiag.InitContext(ctx) logdiag.SetCollect(ctx, true) - phases.LoadNamedTarget(ctx, b, target) + loadErr := phases.LoadNamedTarget(ctx, b, target) diags := logdiag.FlushCollected(ctx) require.Empty(t, diags) + require.NoError(t, loadErr) // Apply initialize / validation mutators - bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { + require.NoError(t, bundle.ApplyFuncContext(ctx, b, func(ctx context.Context, b *bundle.Bundle) { b.Config.Workspace.CurrentUser = &bundleConfig.User{User: cachedUser} - }) + })) b.Tagging = tags.ForCloud(w.Config) b.SetWorkpaceClient(w) b.WorkspaceClient(ctx) - phases.Initialize(ctx, b) + initErr := phases.Initialize(ctx, b) diags = logdiag.FlushCollected(ctx) require.Empty(t, diags) + require.NoError(t, initErr) // Apply build mutator if build { - phases.Build(ctx, b) + _, buildErr := phases.Build(ctx, b) diags = logdiag.FlushCollected(ctx) require.Empty(t, diags) + require.NoError(t, buildErr) } }