Added metrics for immutable folder usage

.nextchanges/bundles/immutable-snapshot-telemetry.md

@@ -0,0 +1 @@
+* Add deploy telemetry for immutable snapshots: whether a deploy used a snapshot and how long its upload took. ([#7168](https://github.com/databricks/cli/pull/7168))

acceptance/bundle/resource_deps/job_tasks/output.txt

@@ -18,6 +18,7 @@ has_classic_interactive_compute false
 has_classic_job_compute false
 has_python_wheel_task true
 has_serverless_compute true
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.miss true
 permissions_section_set false

acceptance/bundle/resource_deps/resources_var/output.txt

@@ -47,6 +47,7 @@ experimental.use_legacy_run_as false
 has_classic_interactive_compute false
 has_classic_job_compute false
 has_serverless_compute false
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.hit true
 permissions_section_set false

acceptance/bundle/resource_deps/tf_path_only_error/out.telemetry.txt

@@ -1,5 +1,6 @@
 experimental.use_legacy_run_as false
 has_tf_only_references true
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.hit true
 presets_name_prefix_is_set false

acceptance/bundle/telemetry/deploy-app-lifecycle-started/output.txt

@@ -93,6 +93,10 @@ Resources: 1 created, 0 changed, 0 deleted, 0 unchanged
       "key": "deployment_history_enabled",
       "value": false
     },
+    {
+      "key": "immutable_snapshot_used",
+      "value": false
+    },
     {
       "key": "app_lifecycle_started",
       "value": true

acceptance/bundle/telemetry/deploy-compute-type/output.txt

@@ -33,6 +33,8 @@ has_serverless_compute true
 has_serverless_compute true
 has_spark_python_task true
 has_spark_python_task true
+immutable_snapshot_used false
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.attempt true
 local.cache.hit true

acceptance/bundle/telemetry/deploy-experimental/output.txt

@@ -20,6 +20,7 @@ has_classic_interactive_compute true
 has_classic_job_compute false
 has_serverless_compute false
 has_spark_python_task true
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.miss true
 permissions_section_set false

acceptance/bundle/telemetry/deploy-immutable-snapshot/databricks.yml

@@ -0,0 +1,20 @@
+bundle:
+  name: test-bundle
+
+experimental:
+  immutable_folder: true
+
+resources:
+  jobs:
+    foo:
+      name: foo
+      tasks:
+        - task_key: task1
+          spark_python_task:
+            python_file: ./src/main.py
+          environment_key: env
+
+      environments:
+        - environment_key: env
+          spec:
+            environment_version: "4"

acceptance/bundle/telemetry/deploy-immutable-snapshot/out.test.toml

@@ -0,0 +1,2 @@
+Cloud = false
+EnvMatrix.DMS = [""]

acceptance/bundle/telemetry/deploy-immutable-snapshot/output.txt

@@ -0,0 +1,26 @@
+
+=== First deploy uploads the snapshot
+
+>>> [CLI] bundle deploy
+Created internal_immutable_snapshots.immutable
+Created jobs.foo
+Files: 0 uploaded, 0 deleted
+Resources: 2 created, 0 changed, 0 deleted, 0 unchanged
+
+>>> print_telemetry_bool_values
+immutable_snapshot_used true
+
+>>> print_snapshot_upload_entries
+1
+
+=== Second deploy finds the snapshot already uploaded
+
+>>> [CLI] bundle deploy
+Files: 0 uploaded, 0 deleted
+Resources: 0 created, 0 changed, 0 deleted, 2 unchanged
+
+>>> print_telemetry_bool_values
+immutable_snapshot_used true
+
+>>> print_snapshot_upload_entries
+0

acceptance/bundle/telemetry/deploy-immutable-snapshot/script

@@ -0,0 +1,17 @@
+# The snapshot upload duration is reported next to the files.(upload) mutator time. Its value
+# varies, so only assert how many times it is reported.
+print_snapshot_upload_entries() {
+    jq -r 'select(.path? == "/telemetry-ext") | (.body.protoLogs // [])[] | fromjson | (.entry // .) | .databricks_cli_log.bundle_deploy_event.experimental.bundle_mutator_execution_time_ms // [] | map(select(.key == "snapshot.upload" and .value >= 0)) | length' "$OUT_REQUESTS"
+}
+
+title "First deploy uploads the snapshot\n"
+trace $CLI bundle deploy
+trace print_telemetry_bool_values | grep immutable_snapshot_used
+trace print_snapshot_upload_entries
+rm "$OUT_REQUESTS"
+
+title "Second deploy finds the snapshot already uploaded\n"
+trace $CLI bundle deploy
+trace print_telemetry_bool_values | grep immutable_snapshot_used
+trace print_snapshot_upload_entries
+rm "$OUT_REQUESTS"

acceptance/bundle/telemetry/deploy-immutable-snapshot/src/main.py

@@ -0,0 +1 @@
+print("hello")

acceptance/bundle/telemetry/deploy-name-prefix/custom/output.txt

@@ -15,6 +15,7 @@ experimental.use_legacy_run_as false
 has_classic_interactive_compute false
 has_classic_job_compute false
 has_serverless_compute false
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.miss true
 permissions_section_set false

acceptance/bundle/telemetry/deploy-name-prefix/mode-development/output.txt

@@ -15,6 +15,7 @@ experimental.use_legacy_run_as false
 has_classic_interactive_compute false
 has_classic_job_compute false
 has_serverless_compute false
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.miss true
 permissions_section_set false

acceptance/bundle/telemetry/deploy-whl-artifacts/output.txt

@@ -35,6 +35,8 @@ has_classic_job_compute false
 has_classic_job_compute false
 has_serverless_compute false
 has_serverless_compute false
+immutable_snapshot_used false
+immutable_snapshot_used false
 local.cache.attempt true
 local.cache.attempt true
 local.cache.hit true

acceptance/bundle/telemetry/deploy/out.telemetry.txt

@@ -125,6 +125,10 @@
             {
               "key": "deployment_history_enabled",
               "value": false
+            },
+            {
+              "key": "immutable_snapshot_used",
+              "value": false
             }
           ],
           "bundle_mode": "TYPE_UNSPECIFIED",

bundle/direct/bundle_apply.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"time"
 
 	"github.com/databricks/cli/bundle/config"
 	"github.com/databricks/cli/bundle/deployplan"
@@ -158,6 +159,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 			//
 			// Success is recorded by the state writes inside Deploy, so a recreate reports
 			// each of its steps.
+			start := time.Now()
 			err = d.Deploy(ctx, &b.StateDB, sv.Value, action, entry)
 			if err != nil {
 				// Empty for a create that never got an ID, and for a recreate whose delete
@@ -167,6 +169,7 @@ func (b *DeploymentBundle) Apply(ctx context.Context, client *databricks.Workspa
 				logdiag.LogError(ctx, fmt.Errorf("%s: %w", errorPrefix, err))
 				return false
 			}
+			b.DeployDurations.Store(resourceKey, time.Since(start))
 
 			// Reported before the remote-state refresh below: the resource is already
 			// deployed at this point, so the line is accurate even if the refresh fails.

bundle/direct/pkg.go

@@ -50,6 +50,11 @@ type DeploymentBundle struct {
 	Plan             *deployplan.Plan
 	RemoteStateCache sync.Map
 	StateCache       structvar.Cache
+
+	// DeployDurations maps a resource key to the time.Duration its create, update or
+	// recreate took. Only successfully deployed resources are recorded; Apply runs
+	// nodes in parallel, hence the sync.Map.
+	DeployDurations sync.Map
 }
 
 // SetRemoteState updates the remote state with type validation and marks as fresh.

bundle/metrics/metrics.go

@@ -12,6 +12,19 @@ const (
 	SqlWarehouseLifecycleStarted        = "sql_warehouse_lifecycle_started"
 	SelectUsed                          = "select_used"
 
+	// Whether the deploy registered an immutable snapshot. Derived from the snapshot
+	// resource, not from experimental.immutable_folder: the flag may change, and the
+	// resource is what the deploy actually uploads.
+	ImmutableSnapshotUsed = "immutable_snapshot_used"
+
+	// Key under which the snapshot upload duration is reported in
+	// bundle_mutator_execution_time_ms, next to files.(upload) so the two can be
+	// compared. Covers creating the snapshot resource (the upload plus recording it in
+	// state), not building the zip, which is snapshot.(snapshotUpload). Absent when no
+	// snapshot was uploaded. It lacks the parentheses of a mutator name because it is
+	// not a mutator.
+	SnapshotUploadTime = "snapshot.upload"
+
 	// Outcome of an automatic migration of terraform state to the direct engine.
 	//   - migrate_error:        state conversion itself errored.
 	//   - migrate_commit_error: the state was converted, but committing it

bundle/phases/telemetry.go

@@ -7,6 +7,7 @@ import (
 	"reflect"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/databricks/cli/bundle"
 	"github.com/databricks/cli/bundle/config"
@@ -199,6 +200,13 @@ func aiRuntimeTaskMetrics(jobs map[string]*resources.Job) (present, scheduled, m
 func LogDeployTelemetry(ctx context.Context, b *bundle.Bundle, errMsg string) {
 	errMsg = telemetry.ScrubErrorMessage(errMsg)
 	b.Metrics.SetBoolValue(metrics.DeploymentHistoryEnabled, b.DeploymentBundle.StateDB.IsDeploymentMetadataService())
+	b.Metrics.SetBoolValue(metrics.ImmutableSnapshotUsed, len(b.Config.Resources.Snapshots) > 0)
+	if d, ok := b.DeploymentBundle.DeployDurations.Load(resources.SnapshotKey); ok {
+		b.Metrics.ExecutionTimes = append(b.Metrics.ExecutionTimes, protos.IntMapEntry{
+			Key:   metrics.SnapshotUploadTime,
+			Value: d.(time.Duration).Milliseconds(),
+		})
+	}
 
 	resourcesCount := int64(0)
 	_, err := dyn.MapByPattern(b.Config.Value(), dyn.NewPattern(dyn.Key("resources"), dyn.AnyKey(), dyn.AnyKey()), func(p dyn.Path, v dyn.Value) (dyn.Value, error) {