diff --git a/bundle/config/mutator/validate_cascade_on_destroy.go b/bundle/config/mutator/validate_cascade_on_destroy.go deleted file mode 100644 index 023b72b991a..00000000000 --- a/bundle/config/mutator/validate_cascade_on_destroy.go +++ /dev/null @@ -1,48 +0,0 @@ -package mutator - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/libs/diag" -) - -type validateCascadeOnDestroy struct { - engine engine.EngineType -} - -// ValidateCascadeOnDestroy returns a mutator that errors when a pipeline sets -// cascade_on_destroy while using the terraform deployment engine. The terraform -// provider does not support the attribute yet, so it is only honored in direct -// deployment mode. -func ValidateCascadeOnDestroy(e engine.EngineType) bundle.Mutator { - return &validateCascadeOnDestroy{engine: e} -} - -func (m *validateCascadeOnDestroy) Name() string { - return "ValidateCascadeOnDestroy" -} - -func (m *validateCascadeOnDestroy) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - if m.engine.IsDirect() { - return nil - } - - var diags diag.Diagnostics - for key, pipeline := range b.Config.Resources.Pipelines { - if pipeline.CascadeOnDestroy == nil { - continue - } - path := "resources.pipelines." + key + ".cascade_on_destroy" - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "cascade_on_destroy is only supported in direct deployment mode", - Detail: "cascade_on_destroy is not yet supported by the terraform provider. " + - "Set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use it.\n" + - "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", - Locations: b.Config.GetLocations(path), - }) - } - return diags -} diff --git a/bundle/config/mutator/validate_cascade_on_destroy_test.go b/bundle/config/mutator/validate_cascade_on_destroy_test.go deleted file mode 100644 index 2af79a69f13..00000000000 --- a/bundle/config/mutator/validate_cascade_on_destroy_test.go +++ /dev/null @@ -1,69 +0,0 @@ -package mutator_test - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/databricks-sdk-go/service/pipelines" - "github.com/stretchr/testify/assert" -) - -func pipelineWithCascade(cascade *bool) *resources.Pipeline { - return &resources.Pipeline{ - CreatePipeline: pipelines.CreatePipeline{Name: "my_pipeline"}, - CascadeOnDestroy: cascade, - } -} - -func TestValidateCascadeOnDestroyDirectEngineReturnsNil(t *testing.T) { - no := false - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Pipelines: map[string]*resources.Pipeline{ - "my_pipeline": pipelineWithCascade(&no), - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineDirect)) - assert.Empty(t, diags) -} - -func TestValidateCascadeOnDestroyTerraformEngineUnsetReturnsNil(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Pipelines: map[string]*resources.Pipeline{ - "my_pipeline": pipelineWithCascade(nil), - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) - assert.Empty(t, diags) -} - -func TestValidateCascadeOnDestroyTerraformEngineSetEmitsError(t *testing.T) { - no := false - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Pipelines: map[string]*resources.Pipeline{ - "my_pipeline": pipelineWithCascade(&no), - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateCascadeOnDestroy(engine.EngineTerraform)) - assert.Len(t, diags, 1) - assert.Equal(t, "cascade_on_destroy is only supported in direct deployment mode", diags[0].Summary) - assert.Contains(t, diags[0].Detail, "https://docs.databricks.com/dev-tools/bundles/direct") -} diff --git a/bundle/config/mutator/validate_cluster_libraries.go b/bundle/config/mutator/validate_cluster_libraries.go deleted file mode 100644 index f13cd3be373..00000000000 --- a/bundle/config/mutator/validate_cluster_libraries.go +++ /dev/null @@ -1,43 +0,0 @@ -package mutator - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/libs/diag" -) - -type validateClusterLibraries struct { - engine engine.EngineType -} - -// ValidateClusterLibraries returns a mutator that errors when cluster libraries are used with -// the terraform deployment engine. Cluster libraries are only supported in direct deployment mode. -func ValidateClusterLibraries(e engine.EngineType) bundle.Mutator { - return &validateClusterLibraries{engine: e} -} - -func (m *validateClusterLibraries) Name() string { - return "ValidateClusterLibraries" -} - -func (m *validateClusterLibraries) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - if m.engine.IsDirect() { - return nil - } - - var diags diag.Diagnostics - for key, cluster := range b.Config.Resources.Clusters { - if cluster == nil || len(cluster.Libraries) == 0 { - continue - } - path := "resources.clusters." + key + ".libraries" - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "cluster libraries are only supported in direct deployment mode", - Locations: b.Config.GetLocations(path), - }) - } - return diags -} diff --git a/bundle/config/mutator/validate_direct_only_resources.go b/bundle/config/mutator/validate_direct_only_resources.go deleted file mode 100644 index 1c3cd41a76c..00000000000 --- a/bundle/config/mutator/validate_direct_only_resources.go +++ /dev/null @@ -1,69 +0,0 @@ -package mutator - -import ( - "context" - "fmt" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/deploy/terraform" - "github.com/databricks/cli/bundle/direct/dresources" - "github.com/databricks/cli/libs/diag" -) - -type validateDirectOnlyResources struct { - engine engine.EngineType -} - -// ValidateDirectOnlyResources returns a mutator that validates resources -// that are only supported in direct deployment mode. -func ValidateDirectOnlyResources(engine engine.EngineType) bundle.Mutator { - return &validateDirectOnlyResources{engine: engine} -} - -func (m *validateDirectOnlyResources) Name() string { - return "ValidateDirectOnlyResources" -} - -// isDirectOnly reports whether a resource type (by PluralName) is supported only -// by the direct engine — present in dresources.SupportedResources but absent -// from terraform.GroupToTerraformName. -func isDirectOnly(pluralName string) bool { - _, hasDirect := dresources.SupportedResources[pluralName] - _, hasTerraform := terraform.GroupToTerraformName[pluralName] - return hasDirect && !hasTerraform -} - -func (m *validateDirectOnlyResources) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - if m.engine.IsDirect() { - return nil - } - - // The state is migrated to the direct engine right after this deploy, so - // direct-only resources are skipped by this run instead of rejected. - // BundleToTerraformWithDynValue logs each resource it skips. - if b.MigratingToDirect { - return nil - } - - var diags diag.Diagnostics - for _, group := range b.Config.Resources.AllResources() { - if len(group.Resources) == 0 { - continue - } - if !isDirectOnly(group.Description.PluralName) { - continue - } - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: group.Description.SingularTitle + " resources are only supported with direct deployment mode", - Detail: fmt.Sprintf("%s resources require direct deployment mode. "+ - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use %s resources.\n"+ - "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", - group.Description.SingularTitle, group.Description.SingularName), - Locations: b.Config.GetLocations("resources." + group.Description.PluralName), - }) - } - - return diags -} diff --git a/bundle/config/mutator/validate_direct_only_resources_test.go b/bundle/config/mutator/validate_direct_only_resources_test.go deleted file mode 100644 index 37c5b679cf4..00000000000 --- a/bundle/config/mutator/validate_direct_only_resources_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package mutator_test - -import ( - "testing" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/mutator" - "github.com/databricks/cli/bundle/config/resources" - "github.com/stretchr/testify/assert" -) - -func TestValidateDirectOnlyResourcesDirectEngineReturnsNil(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Catalogs: map[string]*resources.Catalog{ - "my_catalog": {}, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateDirectOnlyResources(engine.EngineDirect)) - assert.Empty(t, diags) -} - -func TestValidateDirectOnlyResourcesTerraformEngineNoDirectOnlyReturnsNil(t *testing.T) { - b := &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Jobs: map[string]*resources.Job{ - "my_job": {}, - }, - }, - }, - } - - diags := bundle.Apply(t.Context(), b, mutator.ValidateDirectOnlyResources(engine.EngineTerraform)) - assert.Empty(t, diags) -} - -func TestValidateDirectOnlyResourcesTerraformEngineDirectOnlyEmitsError(t *testing.T) { - cases := []struct { - name string - bundle *bundle.Bundle - expectedSummary string - expectedDetail string - }{ - { - name: "catalogs", - bundle: &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - Catalogs: map[string]*resources.Catalog{ - "my_catalog": {}, - }, - }, - }, - }, - expectedSummary: "Catalog resources are only supported with direct deployment mode", - expectedDetail: "Catalog resources require direct deployment mode. " + - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use catalog resources.\n" + - "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", - }, - { - name: "external_locations", - bundle: &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - ExternalLocations: map[string]*resources.ExternalLocation{ - "my_location": {}, - }, - }, - }, - }, - expectedSummary: "External Location resources are only supported with direct deployment mode", - expectedDetail: "External Location resources require direct deployment mode. " + - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use external_location resources.\n" + - "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", - }, - { - name: "vector_search_endpoints", - bundle: &bundle.Bundle{ - Config: config.Root{ - Resources: config.Resources{ - VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ - "my_endpoint": {}, - }, - }, - }, - }, - expectedSummary: "Vector Search Endpoint resources are only supported with direct deployment mode", - expectedDetail: "Vector Search Endpoint resources require direct deployment mode. " + - "Please set the DATABRICKS_BUNDLE_ENGINE environment variable to 'direct' or set 'bundle.engine: direct' in your databricks.yml to use vector_search_endpoint resources.\n" + - "Learn more at https://docs.databricks.com/dev-tools/bundles/direct", - }, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - diags := bundle.Apply(t.Context(), tc.bundle, mutator.ValidateDirectOnlyResources(engine.EngineTerraform)) - if assert.Len(t, diags, 1) { - assert.Equal(t, tc.expectedSummary, diags[0].Summary) - assert.Equal(t, tc.expectedDetail, diags[0].Detail) - } - }) - } -} diff --git a/bundle/config/mutator/validate_lifecycle_started.go b/bundle/config/mutator/validate_lifecycle_started.go deleted file mode 100644 index 67c58a5701c..00000000000 --- a/bundle/config/mutator/validate_lifecycle_started.go +++ /dev/null @@ -1,48 +0,0 @@ -package mutator - -import ( - "context" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/libs/diag" -) - -type validateLifecycleStarted struct { - engine engine.EngineType -} - -// ValidateLifecycleStarted returns a mutator that errors when lifecycle.started -// is used with the terraform deployment engine. -// lifecycle.started is only supported in direct deployment mode. -func ValidateLifecycleStarted(e engine.EngineType) bundle.Mutator { - return &validateLifecycleStarted{engine: e} -} - -func (m *validateLifecycleStarted) Name() string { - return "ValidateLifecycleStarted" -} - -func (m *validateLifecycleStarted) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { - if m.engine.IsDirect() { - return nil - } - - var diags diag.Diagnostics - for _, group := range b.Config.Resources.AllResources() { - for key, resource := range group.Resources { - lws, ok := resource.GetLifecycle().(resources.LifecycleWithStarted) - if !ok || lws.Started == nil { - continue - } - path := "resources." + group.Description.PluralName + "." + key + ".lifecycle.started" - diags = diags.Append(diag.Diagnostic{ - Severity: diag.Error, - Summary: "lifecycle.started is only supported in direct deployment mode", - Locations: b.Config.GetLocations(path), - }) - } - } - return diags -} diff --git a/bundle/direct/resolve_config.go b/bundle/direct/resolve_config.go index bf580413dd1..8ce28d710e1 100644 --- a/bundle/direct/resolve_config.go +++ b/bundle/direct/resolve_config.go @@ -22,10 +22,8 @@ var resourcesPrefix = dyn.MustPathFromString("resources") // unrelated resource may reference a resource that was never deployed, whose ${...}.id // falls back to an empty config string and then fails int type-checking. // -// The state consultation only applies to the direct engine (its state DB holds fields -// like the immutable snapshot's full_path that never reach the config); with terraform -// the state DB is closed and everything resolves from config, which is where state load -// has already written each resource's id. +// The state DB holds fields like the immutable snapshot's full_path that never reach the +// config, so it must be open. func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target dyn.Path) error { return cfg.Mutate(func(root dyn.Value) (dyn.Value, error) { // Fall back to the fully-normalized config so references to fields that are @@ -53,11 +51,6 @@ func (b *DeploymentBundle) ResolveConfigAgainstState(cfg *config.Root, target dy // lookupStateField returns the value at resources... from the // resource's persisted state, if that resource is in state and holds the field. func (b *DeploymentBundle) lookupStateField(path dyn.Path) (dyn.Value, bool) { - // The state DB is only opened for the direct engine; with terraform there is no - // state to consult and references resolve from config alone. - if !b.StateDB.IsOpen() { - return dyn.InvalidValue, false - } if len(path) < 4 || path[0].Key() != "resources" { return dyn.InvalidValue, false } diff --git a/bundle/metrics/metrics.go b/bundle/metrics/metrics.go index 73241aa2882..927bad6b0c0 100644 --- a/bundle/metrics/metrics.go +++ b/bundle/metrics/metrics.go @@ -12,23 +12,11 @@ const ( SqlWarehouseLifecycleStarted = "sql_warehouse_lifecycle_started" SelectUsed = "select_used" - // Outcome of the dry-run migration to the direct engine attempted after a - // successful terraform deploy WHEN THE USER OPTED OUT of direct (direct is - // the default, so this means engine: terraform). Only recorded when the state - // conversion is truly a dry run (no auto-migrate). - // DirectDryMigrateSuccess is false when the state could not be converted; - // DirectDryMigrateWarnings is true when the conversion emitted warnings - // (e.g. resources the direct engine can't represent). - DirectDryMigrateSuccess = "direct_drymigrate_success" - DirectDryMigrateWarnings = "direct_drymigrate_warnings" - - // Outcome of an automatic post-deploy migration to the direct engine, which - // runs unless the user opted out with engine: terraform. These replace the - // direct_drymigrate_* keys on migrating deploys. + // Outcome of an automatic migration of terraform state to the direct engine. // - migrate_error: state conversion itself errored. // - migrate_commit_error: the state was converted, but committing it // (renaming files / pushing to workspace) failed. - // - migrate_warnings: the conversion emitted warnings (see above). + // - migrate_warnings: the conversion emitted warnings (e.g. resources the direct engine can't represent). DirectMigrateError = "direct_migrate_error" DirectMigrateCommitError = "direct_migrate_commit_error" DirectMigrateWarnings = "direct_migrate_warnings" diff --git a/bundle/permissions/terraform_errors.go b/bundle/permissions/terraform_errors.go deleted file mode 100644 index cc4c9f61424..00000000000 --- a/bundle/permissions/terraform_errors.go +++ /dev/null @@ -1,47 +0,0 @@ -package permissions - -import ( - "context" - "fmt" - "regexp" - "strings" - - "github.com/databricks/cli/bundle" - "github.com/databricks/cli/libs/diag" - "github.com/databricks/cli/libs/log" -) - -func TryExtendTerraformPermissionError(ctx context.Context, b *bundle.Bundle, err error) diag.Diagnostics { - _, assistance := analyzeBundlePermissions(b) - - // In a best-effort attempt to provide actionable error messages, we match - // against a few specific error messages that come from the Jobs and Pipelines API. - // For matching errors we provide a more specific error message that includes - // details on how to resolve the issue. - if !strings.Contains(err.Error(), "cannot update permissions") && - !strings.Contains(err.Error(), "permissions on pipeline") && - !strings.Contains(err.Error(), "cannot read permissions") && - !strings.Contains(err.Error(), "cannot set run_as to user") { - return nil - } - - log.Errorf(ctx, "Terraform error during deployment: %v", err.Error()) - - // Best-effort attempt to extract the resource name from the error message. - re := regexp.MustCompile(`databricks_(\w*)\.(\w*)`) - match := re.FindStringSubmatch(err.Error()) - resource := "resource" - if len(match) > 1 { - resource = match[2] - } - - return diag.Diagnostics{{ - Summary: fmt.Sprintf("permission denied creating or updating %s.\n"+ - "%s\n"+ - "They can redeploy the project to apply the latest set of permissions.\n"+ - "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions.", - resource, assistance), - Severity: diag.Error, - ID: diag.ResourcePermissionDenied, - }} -} diff --git a/bundle/permissions/terraform_errors_test.go b/bundle/permissions/terraform_errors_test.go deleted file mode 100644 index 1ad008e251b..00000000000 --- a/bundle/permissions/terraform_errors_test.go +++ /dev/null @@ -1,96 +0,0 @@ -package permissions_test - -import ( - "errors" - "testing" - - "github.com/databricks/cli/bundle/config/resources" - "github.com/databricks/cli/bundle/permissions" - "github.com/databricks/databricks-sdk-go/service/jobs" - "github.com/stretchr/testify/require" -) - -func TestTryExtendTerraformPermissionError1(t *testing.T) { - ctx := t.Context() - b := mockBundle([]resources.Permission{ - {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, - }) - err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ - "\n"+ - "Error: cannot update permissions: ...\n"+ - "\n"+ - " with databricks_pipeline.my_project_pipeline,\n"+ - " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() - - expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + - "For assistance, users or groups with appropriate permissions may include: alice@databricks.com.\n" + - "They can redeploy the project to apply the latest set of permissions.\n" + - "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions" - - require.ErrorContains(t, err, expected) -} - -func TestTryExtendTerraformPermissionError2(t *testing.T) { - ctx := t.Context() - b := mockBundle([]resources.Permission{ - {Level: "CAN_MANAGE", UserName: "alice@databricks.com"}, - {Level: "CAN_MANAGE", UserName: "bob@databricks.com"}, - }) - err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ - "\n"+ - "Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+ - "\n"+ - " with databricks_pipeline.my_project_pipeline,\n"+ - " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() - - expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + - "For assistance, users or groups with appropriate permissions may include: alice@databricks.com, bob@databricks.com.\n" + - "They can redeploy the project to apply the latest set of permissions.\n" + - "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." - require.ErrorContains(t, err, expected) -} - -func TestTryExtendTerraformPermissionError3(t *testing.T) { - ctx := t.Context() - b := mockBundle([]resources.Permission{ - {Level: "CAN_MANAGE", UserName: "testuser@databricks.com"}, - }) - err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ - "\n"+ - "Error: cannot read permissions: 1706906c-c0a2-4c25-9f57-3a7aa3cb8b90 does not have Owner permissions on Job with ID: ElasticJobId(28263044278868). Please contact the owner or an administrator for access.\n"+ - "\n"+ - " with databricks_pipeline.my_project_pipeline,\n"+ - " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() - - expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + - "For assistance, contact the owners of this project.\n" + - "They can redeploy the project to apply the latest set of permissions.\n" + - "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." - require.ErrorContains(t, err, expected) -} - -func TestTryExtendTerraformPermissionErrorNotOwner(t *testing.T) { - ctx := t.Context() - b := mockBundle([]resources.Permission{ - {Level: "CAN_MANAGE", GroupName: "data_team@databricks.com"}, - }) - b.Config.RunAs = &jobs.JobRunAs{ - UserName: "testuser@databricks.com", - } - err := permissions.TryExtendTerraformPermissionError(ctx, b, errors.New("Error: terraform apply: exit status 1\n"+ - "\n"+ - "Error: cannot read pipeline: User xyz does not have View permissions on pipeline 4521dbb6-42aa-418c-b94d-b5f4859a3454.\n"+ - "\n"+ - " with databricks_pipeline.my_project_pipeline,\n"+ - " on bundle.tf.json line 39, in resource.databricks_pipeline.my_project_pipeline:\n"+ - " 39: }")).Error() - - expected := "EPERM2: permission denied creating or updating my_project_pipeline.\n" + - "For assistance, users or groups with appropriate permissions may include: data_team@databricks.com.\n" + - "They can redeploy the project to apply the latest set of permissions.\n" + - "Please refer to https://docs.databricks.com/dev-tools/bundles/permissions.html for more on managing permissions." - require.ErrorContains(t, err, expected) -} diff --git a/bundle/phases/bind.go b/bundle/phases/bind.go index 63241536587..383feb3417b 100644 --- a/bundle/phases/bind.go +++ b/bundle/phases/bind.go @@ -104,7 +104,7 @@ func Bind(ctx context.Context, b *bundle.Bundle, opts *terraform.BindOptions, st return } - statemgmt.PushResourcesState(ctx, b, engine) + statemgmt.PushResourcesState(ctx, b) } func jsonDump(ctx context.Context, v any, field string) string { @@ -147,5 +147,5 @@ func Unbind(ctx context.Context, b *bundle.Bundle, tfResourceType, resourceKey s return } - statemgmt.PushResourcesState(ctx, b, engine) + statemgmt.PushResourcesState(ctx, b) } diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index f7e623f8395..d96f7603b30 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -90,7 +90,7 @@ func deployCore(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan, st } // Even if deployment failed, there might be updates in states that we need to upload - statemgmt.PushResourcesState(ctx, b, stateEngine) + statemgmt.PushResourcesState(ctx, b) if logdiag.HasError(ctx) { return } diff --git a/bundle/phases/plan.go b/bundle/phases/plan.go index f937e2e91c9..3446690fbdd 100644 --- a/bundle/phases/plan.go +++ b/bundle/phases/plan.go @@ -26,10 +26,6 @@ func PreDeployChecks(ctx context.Context, b *bundle.Bundle, isPlan bool, engine resourcemutator.SecretScopeFixups(engine), deploy.StatePull(), mutator.ValidateGitDetails(), - mutator.ValidateDirectOnlyResources(engine), - mutator.ValidateLifecycleStarted(engine), - mutator.ValidateClusterLibraries(engine), - mutator.ValidateCascadeOnDestroy(engine), statemgmt.CheckRunningResource(engine), ) } diff --git a/bundle/statemgmt/direct_migration.go b/bundle/statemgmt/direct_migration.go index 0644ea5c438..a84b5c1f804 100644 --- a/bundle/statemgmt/direct_migration.go +++ b/bundle/statemgmt/direct_migration.go @@ -142,7 +142,7 @@ func CommitMigration(ctx context.Context, b *bundle.Bundle, requiredEngine engin logdiag.LogError(ctx, fmt.Errorf("persisting migrated direct state: %w", err)) return } - PushResourcesState(ctx, b, engine.EngineDirect) + PushResourcesState(ctx, b) if logdiag.HasError(ctx) { return } @@ -171,42 +171,6 @@ func recordAutoMigrateSource(b *bundle.Bundle, requiredEngine engine.EngineSetti } } -// DryRunMigrationTelemetry converts the terraform state to the direct engine WITHOUT -// committing, purely to record direct_drymigrate_* telemetry for deploys that opted out -// of the direct engine (engine: terraform). It mirrors the actual migration's conversion -// (but never runs the plan check, and never touches any state) so the fleet-wide "could -// this bundle migrate?" signal is preserved. It is called after a terraform deploy, so -// mutating b.Config during the conversion is harmless, and it swallows failures because -// the deploy already succeeded. DirectDryMigrateSuccess reflects only whether the state -// conversion succeeded. -func DryRunMigrationTelemetry(ctx context.Context, b *bundle.Bundle) { - _, localTerraformPath := b.StateFilenameTerraform(ctx) - tfState, err := migrate.ParseTFStateFull(ctx, localTerraformPath) - if err != nil { - b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, false) - return - } - if tfState == nil { - return - } - // An empty terraform state has nothing to convert, so the dry run trivially succeeds. - if len(tfState.IDs) == 0 && len(tfState.Attrs) == 0 { - b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, true) - b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, false) - return - } - - tempStatePath, hasWarnings, _, err := convertTFStateToDirect(ctx, b, tfState) - if tempStatePath != "" { - defer func() { - _ = os.Remove(tempStatePath) - _ = os.Remove(tempStatePath + ".wal") - }() - } - b.Metrics.SetBoolValue(metrics.DirectDryMigrateSuccess, err == nil) - b.Metrics.SetBoolValue(metrics.DirectDryMigrateWarnings, hasWarnings) -} - // checkPlanOnTempState opens the migrated state at tempStatePath in read mode, // runs a full plan against it, and returns the plan (and a non-nil error if the // plan fails). Individual planning errors are emitted as warnings with warnPrefix diff --git a/bundle/statemgmt/state_pull.go b/bundle/statemgmt/state_pull.go index 5a721d8abc8..48cfa13aee9 100644 --- a/bundle/statemgmt/state_pull.go +++ b/bundle/statemgmt/state_pull.go @@ -125,12 +125,10 @@ func filerRead(ctx context.Context, f filer.Filer, path string, engine engine.En } // PullResourcesState determines correct state to use by reading all 4 states (terraform/direct, local/remote). -// If state is present and the requested engine disagrees, a warning is issued and the state's engine is used. func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull AlwaysPull, requiredEngine engine.EngineSetting) *StateDesc { var err error - // We read all 4 possible states: terraform/direct X local/remote and then use env var to validate that correct one is used. - // However, states and env var cannot disagree. + // We read all 4 possible states: terraform/direct X local/remote and pick the most recent one. _, localPathDirect := b.StateFilenameDirect(ctx) _, localPathTerraform := b.StateFilenameTerraform(ctx) @@ -162,24 +160,6 @@ func PullResourcesState(ctx context.Context, b *bundle.Bundle, alwaysPull Always return winner } - if requiredEngine.Type != engine.EngineNotSet && requiredEngine.Type != winner.Engine { - // Direct (whether selected explicitly or by default) against a terraform - // state is the auto-migration path; the deploy caller (process.go) prints - // a dedicated pre-deploy hint for that case, so stay quiet here to avoid - // printing the same event twice. - autoMigratePath := requiredEngine.Type == engine.EngineDirect && !winner.Engine.IsDirect() - if !autoMigratePath { - msg := fmt.Sprintf("Deployment engine %q configured in %s does not match the existing state (engine %q). Using %q engine from the existing state.", requiredEngine.Type, requiredEngine.Source, winner.Engine, winner.Engine) - // Warn only when the config also disagrees with the state. If the env var overrides - // a config that matches the state, log at info level to avoid noise. - if requiredEngine.ConfigType != engine.EngineNotSet && requiredEngine.ConfigType != winner.Engine { - logStatesWarning(ctx, msg, states) - } else { - log.Infof(ctx, "%s", msg) - } - } - } - if len(states) == 0 { return winner } @@ -291,10 +271,6 @@ func logStatesError(ctx context.Context, msg string, states []*StateDesc) { logStatesDiag(ctx, diag.Error, msg, states) } -func logStatesWarning(ctx context.Context, msg string, states []*StateDesc) { - logStatesDiag(ctx, diag.Warning, msg, states) -} - func logStatesDiag(ctx context.Context, severity diag.Severity, msg string, states []*StateDesc) { var stateStrs []string for _, state := range states { diff --git a/bundle/statemgmt/state_push.go b/bundle/statemgmt/state_push.go index c7e43d277e0..c000e316e49 100644 --- a/bundle/statemgmt/state_push.go +++ b/bundle/statemgmt/state_push.go @@ -7,7 +7,6 @@ import ( "os" "github.com/databricks/cli/bundle" - "github.com/databricks/cli/bundle/config/engine" "github.com/databricks/cli/bundle/deploy" "github.com/databricks/cli/libs/filer" "github.com/databricks/cli/libs/log" @@ -15,20 +14,14 @@ import ( ) // PushResourcesState uploads the local state file to the remote location. -func PushResourcesState(ctx context.Context, b *bundle.Bundle, engine engine.EngineType) { +func PushResourcesState(ctx context.Context, b *bundle.Bundle) { f, err := deploy.StateFiler(ctx, b) if err != nil { logdiag.LogError(ctx, err) return } - var remotePath, localPath string - - if engine.IsDirect() { - remotePath, localPath = b.StateFilenameDirect(ctx) - } else { - remotePath, localPath = b.StateFilenameTerraform(ctx) - } + remotePath, localPath := b.StateFilenameDirect(ctx) local, err := os.Open(localPath) if errors.Is(err, fs.ErrNotExist) { diff --git a/cmd/bundle/deployment/migrate.go b/cmd/bundle/deployment/migrate.go index 93e229f71e8..3878370d877 100644 --- a/cmd/bundle/deployment/migrate.go +++ b/cmd/bundle/deployment/migrate.go @@ -1,7 +1,6 @@ package deployment import ( - "context" "encoding/json" "errors" "fmt" @@ -83,12 +82,6 @@ to the workspace so that subsequent deploys of this bundle use direct deployment // Same options as regular deploy, to ensure bundle config is in the same state FastValidate: true, Build: true, - PostInitFunc: func(_ context.Context, b *bundle.Bundle) error { - if b.Config.Bundle.Engine == engine.EngineTerraform { - return fmt.Errorf("bundle.engine is set to %q. Migration requires \"engine: direct\" or no engine setting. Change the setting to \"engine: direct\" and retry", engine.EngineTerraform) - } - return nil - }, } b, stateDesc, err := utils.ProcessBundleRet(cmd, opts) diff --git a/cmd/bundle/run.go b/cmd/bundle/run.go index b978f8ee40a..35061f23105 100644 --- a/cmd/bundle/run.go +++ b/cmd/bundle/run.go @@ -181,8 +181,7 @@ Example usage: // Resolve ${resources.*} references within the resource being run so its // runner sees concrete values (e.g. an app's env vars referencing another // resource, or its source_code_path pointing at the immutable snapshot's - // full_path, which lives only in the deployed state). Safe for both engines: - // with terraform the state DB is closed and references resolve from config. + // full_path, which lives only in the deployed state). target := dyn.NewPath(dyn.Key("resources"), dyn.Key(ref.Description.PluralName), dyn.Key(ref.Key)) if err := b.DeploymentBundle.ResolveConfigAgainstState(&b.Config, target); err != nil { return err diff --git a/cmd/bundle/utils/process.go b/cmd/bundle/utils/process.go index cccb1d48911..d11883cde66 100644 --- a/cmd/bundle/utils/process.go +++ b/cmd/bundle/utils/process.go @@ -15,7 +15,6 @@ import ( "github.com/databricks/cli/bundle/config/mutator" "github.com/databricks/cli/bundle/config/validate" "github.com/databricks/cli/bundle/deploy/metadata" - "github.com/databricks/cli/bundle/deploy/terraform" "github.com/databricks/cli/bundle/deployplan" "github.com/databricks/cli/bundle/direct" "github.com/databricks/cli/bundle/direct/dstate" @@ -339,15 +338,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle cmd.SetContext(ctx) } - // --select is only supported by the direct engine, which tracks resource - // dependencies in the plan graph (used to expand the selection transitively). - // Validate once the engine is final (after any migration above), rather than - // silently planning/deploying every resource on terraform. - if len(b.Select) > 0 && !stateDesc.Engine.IsDirect() { - logdiag.LogError(ctx, errors.New("--select is only supported with the direct engine. See https://docs.databricks.com/aws/en/dev-tools/bundles/direct")) - return b, stateDesc, root.ErrAlreadyPrinted - } - // Open direct engine state once for all subsequent operations (ExportState, CalculatePlan, Apply, etc.) // A migrated-from-Terraform state is already open (seeded in memory above), so skip the disk open. needDirectState := stateDesc.Engine.IsDirect() && needsState @@ -427,19 +417,8 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle if opts.ErrorOnEmptyState { modes = append(modes, statemgmt.ErrorOnEmptyState) } - var state statemgmt.ExportedResourcesMap - if stateDesc.Engine.IsDirect() { - state = b.DeploymentBundle.ExportState(ctx) - } else { - var err error - state, err = terraform.ParseResourcesState(ctx, b) - if err != nil { - logdiag.LogError(ctx, err) - return b, stateDesc, root.ErrAlreadyPrinted - } - } mutators := []bundle.Mutator{ - statemgmt.Load(state, modes...), + statemgmt.Load(b.DeploymentBundle.ExportState(ctx), modes...), } // InitializeURLs makes an extra API call; only run it when URLs are needed. if opts.InitIDs { @@ -455,10 +434,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle // precomputed plan. Artifact uploads are handled inside Deploy by extracting remote // paths from the plan's new_state and finding the matching local files. if opts.ReadPlanPath != "" { - if !stateDesc.Engine.IsDirect() { - logdiag.LogError(ctx, errors.New("--plan is only supported with direct engine (set bundle.engine to \"direct\" or DATABRICKS_BUNDLE_ENGINE=direct)")) - return b, stateDesc, root.ErrAlreadyPrinted - } var err error plan, err = deployplan.LoadPlanFromFile(opts.ReadPlanPath) if err != nil { @@ -537,15 +512,6 @@ func ProcessBundleRet(cmd *cobra.Command, opts ProcessOptions) (b *bundle.Bundle return b, stateDesc, root.ErrAlreadyPrinted } } - - // The user opted out of the direct engine (engine: terraform), so no migration ran. - // Do a throwaway conversion of the just-deployed terraform state to record - // direct_drymigrate_* telemetry — the fleet-wide "could this bundle migrate?" - // signal. Runs after the deploy so mutating b.Config during the conversion is - // harmless, and only when the deploy succeeded on a terraform state. - if stateDesc != nil && requiredEngine.Type == engine.EngineTerraform && !stateDesc.Engine.IsDirect() { - statemgmt.DryRunMigrationTelemetry(ctx, b) - } } if opts.PostStateFunc != nil { diff --git a/libs/telemetry/protos/bundle_deploy.go b/libs/telemetry/protos/bundle_deploy.go index 69e01a30715..a3b7d66c6e1 100644 --- a/libs/telemetry/protos/bundle_deploy.go +++ b/libs/telemetry/protos/bundle_deploy.go @@ -48,9 +48,6 @@ type BundleDeployExperimental struct { // Number of configuration files in the bundle. ConfigurationFileCount int64 `json:"configuration_file_count"` - // Size in bytes of the Terraform state file - TerraformStateSizeBytes int64 `json:"terraform_state_size_bytes,omitempty"` - // Number of variables in the bundle VariableCount int64 `json:"variable_count"` ComplexVariableCount int64 `json:"complex_variable_count"` @@ -125,8 +122,7 @@ type BundleResourcesMetadata struct { // Engine that ran the deploy: "direct" or "terraform". StateEngine string `json:"state_engine,omitempty"` - // Size in bytes of the direct engine's resources.json on disk. Terraform - // reports its state file size as Experimental.TerraformStateSizeBytes. + // Size in bytes of the direct engine's resources.json on disk. StateFileSizeBytes int64 `json:"state_file_size_bytes,omitempty"` // One entry per resource type declared in the configuration, plus types that