diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl new file mode 100644 index 00000000000..6549c9af70c --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-bundle-$UNIQUE_NAME + +resources: + features: + foo: + full_name: main.matset_inv_$UNIQUE_NAME.test_feature + source: + delta_table_source: + full_name: main.matset_inv_$UNIQUE_NAME.source_table + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 1w + slide_duration: 24h + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + foo: + materialized_features: + - feature_name: ${resources.features.foo.full_name} + offline_store_config: + catalog_name: main + schema_name: matset_inv_$UNIQUE_NAME + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh new file mode 100755 index 00000000000..80dcb68a26b --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-cleanup.sh @@ -0,0 +1,6 @@ +#!/bin/bash + +# The offline materialization leaves its own tables behind, so the schema is dropped with CASCADE. +echo "Cleaning up temporary source table" +$CLI tables delete main.matset_inv_$UNIQUE_NAME.source_table || true +$CLI schemas delete main.matset_inv_$UNIQUE_NAME --force || true diff --git a/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh new file mode 100755 index 00000000000..18cb15f25ae --- /dev/null +++ b/acceptance/bundle/invariant/configs/materialization_set.yml.tmpl-init.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +# The feature's delta_table_source must already exist, and the invariant harness provisions no +# tables of its own. +echo "Creating temporary source table: main.matset_inv_$UNIQUE_NAME.source_table" + +$CLI schemas create matset_inv_$UNIQUE_NAME main -o json | jq '{full_name}' + +# MSYS_NO_PATHCONV=1 prevents Git Bash on Windows from converting /api/... to a filesystem path. +MSYS_NO_PATHCONV=1 $CLI api post "/api/2.0/sql/statements/" --json "{ + \"warehouse_id\": \"$TEST_DEFAULT_WAREHOUSE_ID\", + \"statement\": \"CREATE TABLE main.matset_inv_$UNIQUE_NAME.source_table (id INT, value STRING, timestamp TIMESTAMP) USING DELTA\", + \"wait_timeout\": \"45s\" + }" > /dev/null diff --git a/acceptance/bundle/resources/features/duration/databricks.yml b/acceptance/bundle/resources/features/duration/databricks.yml new file mode 100644 index 00000000000..84d3654993d --- /dev/null +++ b/acceptance/bundle/resources/features/duration/databricks.yml @@ -0,0 +1,83 @@ +bundle: + name: test-bundle + +variables: + settling: + default: 30m + +resources: + features: + rollingfeat: + full_name: main.myschema.rollingfeat + source: + delta_table_source: + full_name: main.myschema.mysource + lateness: + settling_delay: ${var.settling} + function: + aggregation_function: + count_function: + input: amount + time_window: + rolling: + window_duration: 7d + delay: 1h + entities: + - name: account_id + timeseries_column: + name: event_ts + + slidingfeat: + full_name: main.myschema.slidingfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 1w + slide_duration: 1d + offset: 22h + delay: 15m + entities: + - name: account_id + timeseries_column: + name: event_ts + + sawtoothfeat: + full_name: main.myschema.sawtoothfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sawtooth: + window_duration: 3d15m + entities: + - name: account_id + timeseries_column: + name: event_ts + + tumblingfeat: + full_name: main.myschema.tumblingfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + tumbling: + window_duration: 1d + offset: 6h + entities: + - name: account_id + timeseries_column: + name: event_ts diff --git a/acceptance/bundle/resources/features/duration/out.test.toml b/acceptance/bundle/resources/features/duration/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/features/duration/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/features/duration/output.txt b/acceptance/bundle/resources/features/duration/output.txt new file mode 100644 index 00000000000..4ae314138b3 --- /dev/null +++ b/acceptance/bundle/resources/features/duration/output.txt @@ -0,0 +1,207 @@ + +=== A bundle may write a duration with week, day, hour and minute units +>>> [CLI] bundle summary -o json +{ + "rollingfeat": { + "rolling": { + "delay": "1h", + "window_duration": "7d" + } + }, + "sawtoothfeat": { + "sawtooth": { + "window_duration": "3d15m" + } + }, + "slidingfeat": { + "sliding": { + "delay": "15m", + "offset": "22h", + "slide_duration": "1d", + "window_duration": "1w" + } + }, + "tumblingfeat": { + "tumbling": { + "offset": "6h", + "window_duration": "1d" + } + } +} + +=== The create requests carry the seconds form the API accepts +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.rollingfeat +Created features.sawtoothfeat +Created features.slidingfeat +Created features.tumblingfeat +Files: 1 uploaded, 0 deleted +Resources: 4 created, 0 changed, 0 deleted, 0 unchanged + +>>> print_requests.py --sort //feature-engineering +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.rollingfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "rolling": { + "delay": "3600s", + "window_duration": "604800s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + }, + "lateness": { + "settling_delay": "1800s" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.sawtoothfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "sawtooth": { + "window_duration": "260100s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.slidingfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "sliding": { + "delay": "900s", + "offset": "79200s", + "slide_duration": "86400s", + "window_duration": "604800s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/features", + "body": { + "entities": [ + { + "name": "account_id" + } + ], + "full_name": "main.myschema.tumblingfeat", + "function": { + "aggregation_function": { + "count_function": { + "input": "amount" + }, + "time_window": { + "tumbling": { + "offset": "21600s", + "window_duration": "86400s" + } + } + } + }, + "source": { + "delta_table_source": { + "full_name": "main.myschema.mysource" + } + }, + "timeseries_column": { + "name": "event_ts" + } + } +} + +=== Rewriting a duration in a different unit of the same length plans nothing +>>> update_file.py databricks.yml window_duration: 7d window_duration: 168h + +>>> [CLI] bundle plan -o json +{ + "resources.features.rollingfeat": "skip", + "resources.features.sawtoothfeat": "skip", + "resources.features.slidingfeat": "skip", + "resources.features.tumblingfeat": "skip" +} + +=== An unparseable duration names the field it is on +>>> [CLI] bundle plan --var=settling=7 fortnights +Error: reading config: cannot plan resources.features.rollingfeat: source: lateness: settling_delay: invalid duration "7 fortnights": time: unknown unit " fortnights" in duration "7 fortnights"; use e.g. 604800s, 168h, 7d or 3d15m + + +Exit code: 1 + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.rollingfeat + delete resources.features.sawtoothfeat + delete resources.features.slidingfeat + delete resources.features.tumblingfeat + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 4 deleted diff --git a/acceptance/bundle/resources/features/duration/script b/acceptance/bundle/resources/features/duration/script new file mode 100644 index 00000000000..affe5dd4e9b --- /dev/null +++ b/acceptance/bundle/resources/features/duration/script @@ -0,0 +1,16 @@ +title "A bundle may write a duration with week, day, hour and minute units" +trace $CLI bundle summary -o json | jq -S '.resources.features | map_values(.function.aggregation_function.time_window)' + +title "The create requests carry the seconds form the API accepts" +trace $CLI bundle deploy +trace print_requests.py --sort //feature-engineering + +title "Rewriting a duration in a different unit of the same length plans nothing" +trace update_file.py databricks.yml "window_duration: 7d" "window_duration: 168h" +trace $CLI bundle plan -o json | jq '.plan | with_entries(select(.key | startswith("resources.features"))) | map_values(.action)' + +title "An unparseable duration names the field it is on" +errcode trace $CLI bundle plan --var="settling=7 fortnights" + +trace $CLI bundle destroy --auto-approve +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/README.md b/acceptance/bundle/resources/materialization_sets/README.md new file mode 100644 index 00000000000..f6156c0e553 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/README.md @@ -0,0 +1,281 @@ +# Materialization set acceptance tests + +These run locally against `libs/testserver/materialization_set.go`, which mirrors +`UpdateMaterializationSet`'s reconcile semantics rather than stubbing them. That is deliberate: one +`output.txt` serves both the local run and a `Cloud = true` run, so every rejection message here is +a claim about the real API, and the cloud run is what checks it. One message is not: the unknown +operation in `OperationsGet`, which the backend resolves inside the entity-store client rather than +from `FeatureStoreMessages`. It is marked as such in the fake and no case reaches it. + +## What the update endpoint can do + +The set update reconciles membership and nothing else -- `MutableMaterializedFeatureField.All` is +empty, so every member field is write-once. That shapes most of the cases below: a declared change +to any member field is caught by the planner as a `recreate`, and the only change that reaches the +server is a removal. + +## Create, update and delete are all long-running + +Each write verb returns a `databricks.longrunning.Operation` and the adapter waits on it, so the set +a create or update produces is read off the operation rather than the response body; a delete's +operation settles with no response at all. Execution is synchronous server-side, so the operation is +already terminal when the call returns and the wait settles on its first poll. + +Each verb also takes a `request_id`, and the server reserves an operation record against it before +acting: a repeat carrying a token already used gets the first operation back and changes nothing. +The adapter mints one per call, so the SDK's own retry of a lost response resolves to the first +operation instead of acting twice. The engine's `retryOnTransient` calls the adapter afresh and so +mints a new token; that is safe here because it never retries a create without the adapter opting +in, and re-running an update or a delete reconciles to the same state. No case replays a token, +because every adapter call mints its own. + +## A failure after the operation is reserved is not an HTTP error + +The backend raises only while it can still reject the request outright, and that boundary sits +earlier than "every validation". On the create path only three rules are reached before the record +is reserved: a member list that is empty, a supplied set id that already exists, and the schedule +states agreeing, which a wire-level validation hook checks before the handler runs at all. The +batch-and-streaming mixing rule, both destination rules and the sawtooth triple rule all live inside +the batch create the handler calls *after* reserving, so each of them settles the record and comes +back as HTTP 200. Measured against the live build, not read off the handler: `batch` and `sawtooth` +each failed on a golden still expecting a 400. Past that point a failure is the operation's outcome, +so all three verbs settle the record FAILED and return it with HTTP 200, carrying the error code and +message on the operation and the per-member outcome on the metadata's progress arm. A creation has +no progress arm: it has no per-member outcome to report. + +The generated wait flattens that into one string and reads no metadata, so the adapter re-reads the +settled operation to recover the code structurally and appends the members the change left alone. +It returns an `APIError` so the engine classifies and renders the failure as it would any other, +with the 200 the call actually returned as the status -- which maps to no sentinel, deliberately: a +member that could not be torn down must not reach the destroy step's permission-denied leniency, +whose purpose is to forgive a resource that is already gone. + +One gap this leaves is the engine's, not ours, and `settled_failure` pins it: `DeploymentUnit.Delete` +forgives a delete error matching none of the three cases it knows, so a partly deleted set exits +zero with its state dropped. The deployment-history leg is the one path that records and returns it. + +## What each case pins + +| Case | Claim | +|---|---| +| `basic` | create through the long-running operation, then a redeploy that plans nothing -- the server-assigned schedule state and the output-only member fields must not read as drift; the destroy's operation round trip | +| `bind` | unbinding leaves the set in the workspace and the plan reports `create`, which is the out-of-band shape; binding it back settles the first plan to `skip`, with the destination suppressed exactly as on a bundle-created set | +| `membership` | the delete cascade runs both ways -- removing either member of a feature's pair on its own is refused, while removing a whole pair plans as `update` and the other feature's pair keeps its ids | +| `removal_scope` | the cascade is scoped per feature: where every feature is materialized offline only, each contributes one member and can have it removed alone; the update's operation round trip | +| `schedule_state` | a member the config leaves unset adopts the state the server assigned; a declared change to it is a `recreate`, and that recreate still has to satisfy the create endpoint's rule that a feature's members agree | +| `immutable` | a write-once member field is caught by the planner as `recreate` and never reaches the server; adding a member routes to `update` so the server refuses it by name; destroy then redeploy is the route those refusals leave open, and the re-created set plans clean | +| `destination` | a destination field the server accepts on write and never returns on read is not drift, while a repoint the config actually asked for still recreates -- the two are told apart by whether the persisted value is recoverable | +| `batch` | a batch feature with both destinations creates and settles; neither member of its pair can be removed alone, and removing both empties the set, which the update endpoint refuses in favour of delete; repointing the destinations is a `recreate`; pausing the feature is a `recreate` that succeeds when every member agrees and is refused by the create endpoint when they do not; a set mixing a batch-path feature with a streaming-path one is refused at create | +| `sawtooth` | the offline-and-online batch pair is refused at create, because a set is not expanded into the third materialization; all three declared creates and then plans nothing; the three members are one cascade group, so each of the three strict subsets of them is refused -- with the message depending on whether the removal names the member a delete is applied to -- and removing all three empties the set, which the update endpoint refuses | +| `settled_failure` | a failure the server hits after reserving an operation comes back as HTTP 200 with the operation FAILED: the adapter recovers the error code from the operation and the per-member outcome from its metadata, the update fails naming each member left behind and writes nothing, and the destroy reports success anyway because the engine forgives a delete error it cannot classify | +| `shorthand` | a feature's whole materialization declared once in `sawtooth_materialized_features` expands into its three members, and in `batch_offline_and_online_materialized_features` into its two; neither authoring field survives into the summary, a redeploy plans nothing, and one edit to the online destination reaches both online members while the offline one stays put. A batch schedule the service refuses is reported by the service naming the feature, not by the CLI -- the expansion judges nothing it is given. One arm mixes a shorthand entry with members written out in full, so the expansion is appended rather than substituted; another carries the batch pair shorthand alone, with no member written out. The last declares both authoring fields in one set, which the service refuses because a sawtooth entry has to carry a DERIVED schedule and a batch pair a MANUAL one, and one request cannot mix cron schedule modes. Its fixtures write the feature windows three ways: seconds only, extended units alone (`3d`), and units mixed (`2d1h35m40s`) | +| `streaming` | a streaming online member needs no schedule and no cron; declaring a schedule state on it, or repointing its destination, plans as a `recreate` rather than an in-place update; with two features materialized to one destination and no offline member between them, each online member is its own cascade group, so one can be removed alone as an `update` | + +## Running against a workspace + +`batch`, `sawtooth`, `streaming` and `shorthand` reference an existing `Stream`, because a Stream +is not a bundle resource. **The `MATSET_STREAM` default names one developer's Stream and goes stale +when it is deleted**, which fails the run at the feature create with `Stream '...' not found`; list +what the workspace has with `GET /api/2.0/feature-engineering/streams?parent=.` and +override. Override `MATSET_CATALOG`, `MATSET_SCHEMA`, +`MATSET_ONLINE_STORE` and `MATSET_STREAM` to move them to another workspace, and set +`MATSET_PROFILE` to pin the bundle to a named profile. **`DATABRICKS_CONFIG_PROFILE` does not reach +the CLI subprocess**, and the harness builds its own workspace client from the ambient environment +before any script runs, so a local cloud run needs working ambient auth to the target workspace. +`batch` also needs `TEST_DEFAULT_WAREHOUSE_ID`, which the harness only supplies off cloud. + +The deployment-history leg (`DMS=true`) cannot run **against a workspace**: the bundle deployments +API rejects `features.` and `materialization_sets.` as resource-type prefixes, so every deploy of +either type fails there with `operations.resource_key must have a known resource type prefix`. +Select the `DMS=` leg for a cloud run until that allowlist includes them. The local `DMS=true` leg +does run, because the fake only requires the key to carry a prefix at all. + +### Running a cloud leg + +Seven cases carry `Cloud = true`: `batch`, `sawtooth`, `streaming`, `immutable`, `membership`, +`removal_scope` and `shorthand`. `basic`, `bind`, `destination` and `schedule_state` stay local, because every +server-dependent claim they make is already exercised by `batch`. + +The cases default to the shared acceptance schema; override `MATSET_CATALOG`, `MATSET_SCHEMA`, +`MATSET_ONLINE_STORE` to move them. These reach the script because the harness forwards ambient +environment to the CLI subprocess, minus the SDK's own auth variables, which it replaces with the +ones it resolved. + +```bash +TOK=$(databricks auth token -p -o json | jq -r .access_token) +DATABRICKS_CONFIG_FILE=/dev/null DATABRICKS_HOST= DATABRICKS_TOKEN="$TOK" CLOUD_ENV=aws TEST_DEFAULT_WAREHOUSE_ID= DATABRICKS_LITESWAP_TARGET= go test . -run 'TestAccept/bundle/resources/materialization_sets//DMS=$' +``` + +Four parts of that are load-bearing: + +- **`DATABRICKS_CONFIG_FILE=/dev/null`.** With `RecordRequests = true` the harness proxies the run + and hands the subprocess a config carrying the *resolved config-file path*. The subprocess then + re-reads it and can fall back to `[__settings__] default_profile`, whose credentials have nothing + to do with the workspace under test. The symptom is a token refresh against + `/oidc/accounts//v1/token`. +- **A token rather than a profile.** The harness flattens its resolved auth into environment + variables, which overrides a `workspace: profile:` pinned in the bundle. A profile whose + `auth_type` is `databricks-cli` therefore gets re-resolved in the subprocess without the profile. +- **`/DMS=$`.** The `DMS=true` leg cannot run against a workspace: the bundle deployments API + rejects `features.` and `materialization_sets.` as resource-type prefixes. +- **The unit name only** in `DATABRICKS_LITESWAP_TARGET` -- `liteswap.Apply` prepends + `testenv://liteswap/`. Passing the full traffic id routes to base staging, where these endpoints + do not exist. + +The workspace needs a declarative feature preview enabled -- `validateAnyDeclarativePreviewEnabled` +rejects every create otherwise, with `declarativePreviewsNotEnabled`. + +### Two destination rules the fixtures have to satisfy + +Both are enforced on create by `BatchMaterializationUtils`, and both are now mirrored in the fake, +so an invalid fixture fails locally instead of only on cloud. + +1. **Every feature in a set must ask for an identical destination *pair*.** The compared value is + `(offlineStoreConfig, Option[onlineStoreConfig])` per feature, so matching catalog, schema and + prefix is not enough: a set cannot mix a feature materialized to both stores with one + materialized offline only. That is why `membership` gives every feature a pair and + `removal_scope` gives every feature an offline member alone -- the rule makes those two shapes + mutually exclusive within one set, which is why they cannot be one case. +2. **A feature's offline and online destinations must differ** in catalog, schema or prefix. The + fixtures suffix the offline prefix with `_off`. + +A third thing the fake does not check: that a member's feature exists. Declare the `features` +resources and reference them as `${resources.features..full_name}`, which also orders the +feature before the set -- a bare name leaves the create racing the feature, which is what made +`sawtooth` flaky. + +## Cloud runs 2026-10-20 + +Run against the `daniel-bai-fs2680` swap on workspace 2978662807315490. Two swap builds, and the +difference between them is the point. + +**First build, universe `afc23c250043d`.** `batch`, `sawtooth`, `streaming` and `immutable` passed; +`membership` and `removal_scope` failed, both refused with: + +``` +The fields 'pipeline_schedule_state' of materialized feature '' of feature '' cannot be +changed. Only removing members is supported; recreate the materialization set instead. +``` + +`comparableMaterializationSetMembers` copied the requested `pipelineScheduleState` raw onto the +comparison baseline while normalising the trigger beside it, so a bundle -- which never declares +the field, it being a backend default -- compared `None` against the stored `ACTIVE` and had every +membership change refused. Isolated live: the same removal succeeded when the surviving member +carried `pipeline_schedule_state: ACTIVE` and failed when it did not, nothing else different. + +**Second build, universe `8438b2287fb03`.** Fixed, and the validations moved into +`MaterializationSetUtils`. An omitted state now falls back to the stored one +(`.orElse(normalizedExistingMaterializedFeature.pipelineScheduleState)`), and the changed-field list +is derived from `MaterializedFeature.scalaDescriptor.fields` rather than a hand-maintained `Seq`, so +a member field added later is covered without touching this check. + +All six cases have passed against that build. The one golden this cost was `membership`: the cascade +refusal was reworded, and the fake carried the old text. + +``` +old: Changing one materialized feature of feature 'F' also changes '', which the request does + not change. Apply the same change to every materialized feature of the feature. +new: Changing materialized features '' of feature 'F' also changes materialized features + '', which the request does not change. Apply the same change to the unrequested + materialized features as well. +``` + +The new message names the removed members of the offending feature, so the fake groups removed +members by feature and sorts the ids the way `FeatureStoreMessages.quotedIds` renders them. +`materializedFeatureFieldCannotBeChanged` was reworded to match the server too, though no case +reaches it -- the planner classifies every member field as a recreate, so a declared change never +becomes an update request. + +**The workspace throws intermittent 500s that are not this resource.** Across five runs of the +`DMS=` leg, individual cases failed on `POST /api/2.0/feature-engineering/features` with `Unable to +create Feature '' in UC. If the issue persists, please retry or contact support.`, and once on +`Failed to create Lakebase table`. `immutable` lost three runs in a row this way, then passed twice in +a row when run on its own -- so the trigger is the parallel load of six bundles deploying at once, +not the fixture. Its two features are identical but for their names, and which one fails varies; a +direct two-feature create reproduced nothing, serial or concurrent. Run a case on its own before +reading a red run as a regression. + +**Third build, universe `275193b7facc8`.** The deletion cascade refusal became two messages, +picked by whether the removal names the member a delete is applied to: +`materializationSetMembersCannotBeRemovedAlone` when it does not, and +`deletionAffectsUnrequestedMaterializedFeatureMembers` when it does. The fake branches the same way +and `membership`'s two arms now show one each. A set mixing a batch-path feature with a +streaming-path one is also refused, which the fake had not modelled; `batch`'s last arm pins it, +with the message captured from this build. + +All six `Cloud = true` cases pass against this build, run one at a time, with no golden changed -- +so every message in the removal matrix is the server's own text and not a reading of its source. +That covers both cascade messages, the empty-set refusal that sends a caller to the delete endpoint, +the batch-versus-streaming mixing refusal, and the `streaming` case's claim that two stream features +materialized to one destination can share a set, which until then rested on the shared-destination +rule comparing a `(offline, Option[online])` pair that is identical for both. + +**Fourth build, universe `cd7a68312ca3`** -- the promotion tip, carrying the settled-failure +contract. All six `Cloud = true` cases pass against it, run one at a time. Three messages had been +reworded (`materializationSetMembersCannotBeRemovedAlone`, and both partial-change messages, which +now count what got through and point at the operation's progress), and two refusals moved from an +HTTP 400 to a settled FAILED operation. The fake and four goldens were corrected from what the live +run returned. + +**Fifth build, universe `92bd718e722a0`** -- the tip of `stack/fs2680-update-set-lro`, run through +the `daniel-bai-fs2680` swap on 2026-10-23. All seven `Cloud = true` cases pass with no golden +changed, run one at a time, as does `features/lifecycle` on its `DMS=` leg. Two of them had claims +nothing live had settled: `shorthand` had never run against a workspace at all, and `batch` carried +two arms -- the null member-list spelling and the drop-the-resource delete route -- that had only +run against the fake. So the emptied-set refusal's "Delete the set instead" is the service's own +text, and the one-segment `materialized_features` path classifies as an update on the live API and +not just in the unit tests. + +`shorthand` is the case that run first justified: a sawtooth entry creates the three members the +service wants, a redeploy plans nothing, one edit to the online destination recreates exactly the +two online members, a shorthand entry and members written out in full coexist in one set, and the +DERIVED-schedule refusal its first arm asserts is the service's own text. + +The swap's own log is what proves the routing: during the 63-second run it served 5 +`CreateMaterializationSetRequest`, 7 `DeleteMaterializationSetRequest`, 6 +`GetMaterializationSetRequest`, 15 `ListMaterializationSetMembersRequest`, 3 +`CreateMaterializationSetMembersRequest` and 4 `DeleteMaterializedFeatureRequest`, interleaved with +this run's own fixture names. Zero `UpdateMaterializationSetRequest`, which is the case's one +coverage limit: every membership edit it makes is to an immutable field, so it plans a recreate and +never reaches the update endpoint. `batch`'s emptied-member arms are what exercise that. + +## Verified against a workspace, with no case of their own + +Run through the `daniel-bai-fs2680` swap on 2026-10-19, against a build carrying the write-once +schedule state. Every `Cloud = true` case passed on that run, so their goldens -- including +`batch`'s recreate carrying `PAUSED` on both members -- are confirmed against the real API. + +- **Creation across member kinds.** A batch feature with both destinations; two features sharing one + destination with a pair each; a sawtooth triple; a lone streaming RTM member. The sad paths too: + the sawtooth pair alone, members of one feature disagreeing on schedule state, two features with + different destinations, MBM on a non-Kinesis source, and an empty body. +- **Out-of-band creation is caught by the server, not by the planner.** The plan says `create` + because the bundle has no state for the resource; the deploy then fails with `ALREADY_EXISTS` on + the feature, and a set whose feature already has materializations fails with + `RESOURCE_ALREADY_EXISTS`. A set has no name to collide on, so that second check is the only thing + standing between an unbound set and a duplicate. The `bind` case pins the planner half of this + locally; the server half needs a workspace. +- **Get redacts the offline destination.** All four members of a two-feature set read back with + `offline_store_config` absent, which is what the `destination` suppression exists to absorb. + +Two behaviours previously recorded here are gone, not untested: a mixed update whose halves are both +legal, and the atomicity of a mixed update when one half is rejected. Both depended on a pause being +a legal update. With no mutable member field left, the only update is a removal, so there is no +mixed update to construct. + +## Not covered yet + +- **A set's materialized output tables outlive it.** `bundle destroy` removes the set and its + features, leaving each offline materialization's `*_partial_aggregates` table behind. A cloud case + that wants a clean schema has to drop those itself. + +- **A real member-teardown failure.** `settled_failure` produces the settled-FAILED shape + from a marker the fake looks for in a feature name, because a real teardown fails on + infrastructure no request can ask for. The adapter's reading of that shape is pinned; that + the server emits it for a genuine failure is not, and cannot be from here. + +- **A settled-but-failed operation.** Every refusal these cases reach is an HTTP error on the call + itself, raised before the operation record exists, so the wait is never what fails. The branch + that reports a failed operation -- naming the set from the operation metadata -- is covered only + by `TestMaterializationSetOperationTarget`, and the same injection the partial-failure case needs + would cover it end to end. diff --git a/acceptance/bundle/resources/materialization_sets/basic/databricks.yml b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml new file mode 100644 index 00000000000..4473f2068f8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/databricks.yml @@ -0,0 +1,45 @@ +bundle: + name: test-bundle + +resources: + features: + batchfeat: + full_name: main.myschema.batchfeat + source: + delta_table_source: + full_name: main.myschema.mysource + function: + aggregation_function: + count_function: + input: amount + time_window: + sliding: + window_duration: 7d + slide_duration: 1d + entities: + - name: account_id + timeseries_column: + name: event_ts + + materialization_sets: + set1: + materialized_features: + # One feature with both destinations: the offline member a delete cascades from, and the + # online member it cascades to. + - feature_name: main.myschema.batchfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.batchfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: batchfeat + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/basic/out.test.toml b/acceptance/bundle/resources/materialization_sets/basic/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/basic/output.txt b/acceptance/bundle/resources/materialization_sets/basic/output.txt new file mode 100644 index 00000000000..2fe3d5fb683 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/output.txt @@ -0,0 +1,115 @@ + +=== Summary before deploy +>>> [CLI] bundle summary -o json +{ + "modified_status": "created", + "materialized_features": [ + { + "feature_name": "main.myschema.batchfeat", + "is_online": null + }, + { + "feature_name": "main.myschema.batchfeat", + "is_online": null + } + ] +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created features.batchfeat +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +=== The create returns a long-running operation, and the created set is read back from it +>>> print_requests.py --keep //materialization-sets +{ + "method": "POST", + "path": "/api/2.0/feature-engineering/materialization-sets", + "q": { + "request_id": "[UUID]" + }, + "body": { + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.batchfeat", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "myschema", + "table_name_prefix": "offline" + } + }, + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.myschema.batchfeat", + "online_store_config": { + "catalog_name": "main", + "online_store_name": "danielb", + "schema_name": "myschema", + "table_name_prefix": "batchfeat" + } + } + ] + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} + +=== The server assigned a set id and a member id per member +>>> [CLI] bundle summary -o json +{ + "id": "[UUID]" +} + +=== Redeploying with no config change must plan nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 0 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +>>> print_requests.py //materialization-sets + +=== The schedule state the server assigned must not read as drift +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +=== The delete returns a long-running operation too, which the destroy waits on +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.batchfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 2 deleted + +>>> print_requests.py --keep //materialization-sets +{ + "method": "DELETE", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]" + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} diff --git a/acceptance/bundle/resources/materialization_sets/basic/script b/acceptance/bundle/resources/materialization_sets/basic/script new file mode 100644 index 00000000000..cd50c2da629 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/basic/script @@ -0,0 +1,26 @@ +title "Summary before deploy" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {modified_status, materialized_features: [.materialized_features[] | {feature_name, is_online}]}' + +trace $CLI bundle deploy + +title "The create returns a long-running operation, and the created set is read back from it" +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + +title "The server assigned a set id and a member id per member" +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | {id}' + +title "Redeploying with no config change must plan nothing" +trace $CLI bundle deploy +trace print_requests.py //materialization-sets + +title "The schedule state the server assigned must not read as drift" +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json + +title "The delete returns a long-running operation too, which the destroy waits on" +trace $CLI bundle destroy --auto-approve +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl new file mode 100644 index 00000000000..4d80c657479 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/databricks.yml.tmpl @@ -0,0 +1,46 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 6d24h + slide_duration: 23h60m + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl new file mode 100644 index 00000000000..8c72a5c0c3c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-both-null.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl new file mode 100644 index 00000000000..234b08d05e6 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-both.yml.tmpl @@ -0,0 +1,26 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: [] diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl new file mode 100644 index 00000000000..587d203b3bc --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-offline.yml.tmpl @@ -0,0 +1,35 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl new file mode 100644 index 00000000000..032124311b0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/drop-online.yml.tmpl @@ -0,0 +1,36 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl new file mode 100644 index 00000000000..9bba1ce5161 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/mixed.yml.tmpl @@ -0,0 +1,63 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + # Stream-sourced, so it materializes on the streaming path while batchfeat stays on the + # batch path. The set below asks for one of each. + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batchstream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batchstream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl new file mode 100644 index 00000000000..b5eb76c26b2 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/moved.yml.tmpl @@ -0,0 +1,46 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from, and the only one a schedule state can be + # applied to. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: moved_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: moved_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl new file mode 100644 index 00000000000..ed86dd96e53 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/no-set.yml.tmpl @@ -0,0 +1,22 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp diff --git a/acceptance/bundle/resources/materialization_sets/batch/out.test.toml b/acceptance/bundle/resources/materialization_sets/batch/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/batch/output.txt b/acceptance/bundle/resources/materialization_sets/batch/output.txt new file mode 100644 index 00000000000..a2fe2d0890e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/output.txt @@ -0,0 +1,221 @@ + +>>> create_table.py danielb.fs2680.src_[UNIQUE_NAME] +Created table danielb.fs2680.src_[UNIQUE_NAME] +Inserted sample data into danielb.fs2680.src_[UNIQUE_NAME] +Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) + +=== A batch feature with both destinations creates, and a redeploy plans nothing +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created features.batchfeat +Created materialization_sets.set1 +Files: 11 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + +=== Neither member of the pair can be removed on its own. The offline member is the one a +=== delete is applied to, so removing it reaches the online one too, +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== and removing the online member names nothing a delete can be applied to +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.batch_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. + +Files: 2 uploaded, 0 deleted + +=== Removing both satisfies the cascade but empties the set, which the update endpoint refuses +=== in favour of the delete endpoint. Restoring the pair plans nothing, so no refusal wrote. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +=== Writing no value instead of an empty list is the same declaration, and has to reach the same +=== endpoint: the member field emptied whole is one path rather than one per member, and the +=== immutable rules match it just as readily. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + +=== Taking the refusal's advice: dropping the set resource rather than its members plans as a +=== delete and settles. This is the deploy path's delete, not bundle destroy's. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "delete" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + delete resources.materialization_sets.set1 +Deleted materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "create" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +=== Repointing the destinations is a recreate, not a silent no-op. The server cannot move a +=== materialization in place, so the set is rebuilt at the new location. +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)" out.plan.json +spec:immutable old=batch_[UNIQUE_NAME]_off new=moved_[UNIQUE_NAME]_off +spec:immutable old=batch_[UNIQUE_NAME] new=moved_[UNIQUE_NAME] + +=== The schedule state is write-once, so pausing the feature is a recreate rather than the +=== in-place update the endpoint would refuse +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "skip" +} + +=== Pausing the offline member alone plans as a recreate too, and the create refuses the +=== members of one feature disagreeing on the state +>>> [CLI] bundle plan -o json + +>>> set_actions +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: cannot recreate resources.materialization_sets.set1: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. + +Files: 2 uploaded, 0 deleted + +=== A set cannot mix a feature that materializes on the batch path with one that materializes +=== on the streaming path. The refusal above left no set behind, so this reaches the create +=== endpoint -- which reserves its operation record before it reaches this rule, so the refusal +=== settles that record and arrives as HTTP 200 rather than as an error on the call. +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-batch-[UNIQUE_NAME]/default/files... +Created features.streamfeat +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] (200 INVALID_PARAMETER_VALUE) + +Endpoint: n/a +HTTP Status: 200 +API error_code: INVALID_PARAMETER_VALUE +API message: materialization set [UUID] operation failed: Batch and streaming features cannot be mixed in a single materialize_features call because they require different trigger types. Issue two separate calls: one for batch features and one for streaming features. Batch features: danielb.fs2680.batch_[UNIQUE_NAME]. Streaming features: danielb.fs2680.batchstream_[UNIQUE_NAME] + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl new file mode 100644 index 00000000000..274bc48785f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/paused-both.yml.tmpl @@ -0,0 +1,47 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl new file mode 100644 index 00000000000..26b239c109f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/paused-offline.yml.tmpl @@ -0,0 +1,46 @@ +bundle: + name: test-matset-batch-$UNIQUE_NAME + +resources: + features: + batchfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.batch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + # The offline member a delete cascades from. + - feature_name: ${resources.features.batchfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_${UNIQUE_NAME}_off + pipeline_schedule_state: PAUSED + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + # The online member the delete cascades to. + - feature_name: ${resources.features.batchfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: batch_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/batch/script b/acceptance/bundle/resources/materialization_sets/batch/script new file mode 100644 index 00000000000..62624214c43 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/script @@ -0,0 +1,105 @@ +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the fixture a phase deploys is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_actions() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +title "A batch feature with both destinations creates, and a redeploy plans nothing" +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +title "Neither member of the pair can be removed on its own. The offline member is the one a" +title "delete is applied to, so removing it reaches the online one too," +render drop-offline.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "and removing the online member names nothing a delete can be applied to" +render drop-online.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "Removing both satisfies the cascade but empties the set, which the update endpoint refuses" +title "in favour of the delete endpoint. Restoring the pair plans nothing, so no refusal wrote." +render drop-both.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "Writing no value instead of an empty list is the same declaration, and has to reach the same" +title "endpoint: the member field emptied whole is one path rather than one per member, and the" +title "immutable rules match it just as readily." +render drop-both-null.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +title "Taking the refusal's advice: dropping the set resource rather than its members plans as a" +title "delete and settles. This is the deploy path's delete, not bundle destroy's." +render no-set.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy + +title "Repointing the destinations is a recreate, not a silent no-op. The server cannot move a" +title "materialization in place, so the set is rebuilt at the new location." +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.action == "recreate")) | .[] | "\(.value.reason) old=\(.value.old) new=\(.value.new)"' out.plan.json + +title "The schedule state is write-once, so pausing the feature is a recreate rather than the" +title "in-place update the endpoint would refuse" +render paused-both.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_actions + +title "Pausing the offline member alone plans as a recreate too, and the create refuses the" +title "members of one feature disagreeing on the state" +render paused-offline.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_actions +trace musterr $CLI bundle deploy --auto-approve + +title "A set cannot mix a feature that materializes on the batch path with one that materializes" +title "on the streaming path. The refusal above left no set behind, so this reaches the create" +title "endpoint -- which reserves its operation record before it reaches this rule, so the refusal" +title "settles that record and arrives as HTTP 200 rather than as an error on the call." +render mixed.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/batch/test.toml b/acceptance/bundle/resources/materialization_sets/batch/test.toml new file mode 100644 index 00000000000..ac04768e226 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/batch/test.toml @@ -0,0 +1,29 @@ +# Pinned to a workspace holding a Stream, since a Stream is not a bundle resource. Override +# MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/bind/databricks.yml b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml new file mode 100644 index 00000000000..0a956c87748 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/databricks.yml @@ -0,0 +1,18 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.bindfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: shared_off + - feature_name: main.myschema.bindfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: shared diff --git a/acceptance/bundle/resources/materialization_sets/bind/out.test.toml b/acceptance/bundle/resources/materialization_sets/bind/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/bind/output.txt b/acceptance/bundle/resources/materialization_sets/bind/output.txt new file mode 100644 index 00000000000..a5133235426 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/output.txt @@ -0,0 +1,53 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== Unbinding leaves the set in the workspace and drops it from the bundle's state, which is +=== the same shape as a set created out of band: the plan reports create, because the bundle +=== has nothing recorded for it +>>> [CLI] bundle deployment unbind set1 + +>>> [CLI] bundle plan -o json + +>>> set_plan +{ + "resources.materialization_sets.set1": "create" +} + +=== Binding the existing set takes ownership of it again +>>> [CLI] bundle deployment bind set1 [UUID] --auto-approve +Successfully bound materialization_set with an id '[UUID]' +Run 'bundle deploy' to deploy changes to your workspace + +=== The first plan after a bind settles. Bind seeds state from the resolved config rather +=== than from a read, so the destination the read never returns is suppressed here exactly as +=== it is on a set the bundle created +>>> [CLI] bundle plan -o json + +>>> set_plan +{ + "resources.materialization_sets.set1": "skip" +} + +>>> destination_changes +materialized_features[member='feature_name=main.myschema.bindfeat|is_streaming_mode=false'].offline_store_config skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name skip input_only_not_returned +materialized_features[member='feature_name=main.myschema.bindfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix skip input_only_not_returned + +=== And the deploy is a no-op rather than a rebuild +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 1 unchanged + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-bundle/default + +Destroy: 1 deleted diff --git a/acceptance/bundle/resources/materialization_sets/bind/script b/acceptance/bundle/resources/materialization_sets/bind/script new file mode 100644 index 00000000000..fa69819fc52 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/script @@ -0,0 +1,36 @@ +set_plan() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +destination_changes() { + jq -r '.plan["resources.materialization_sets.set1"].changes // {} | to_entries + | map(select(.key | test("store_config"))) + | map("\(.key) \(.value.action) \(.value.reason)") | .[]' out.plan.json +} + +trace $CLI bundle deploy +set_id=$($CLI bundle summary -o json | jq -r '.resources.materialization_sets.set1.id') + +title "Unbinding leaves the set in the workspace and drops it from the bundle's state, which is" +title "the same shape as a set created out of band: the plan reports create, because the bundle" +title "has nothing recorded for it" +trace $CLI bundle deployment unbind set1 +trace $CLI bundle plan -o json > out.plan.json +trace set_plan + +title "Binding the existing set takes ownership of it again" +trace $CLI bundle deployment bind set1 "$set_id" --auto-approve + +title "The first plan after a bind settles. Bind seeds state from the resolved config rather" +title "than from a read, so the destination the read never returns is suppressed here exactly as" +title "it is on a set the bundle created" +trace $CLI bundle plan -o json > out.plan.json +trace set_plan +trace destination_changes + +title "And the deploy is a no-op rather than a rebuild" +trace $CLI bundle deploy + +trace $CLI bundle destroy --auto-approve + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/bind/test.toml b/acceptance/bundle/resources/materialization_sets/bind/test.toml new file mode 100644 index 00000000000..83e04113992 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/bind/test.toml @@ -0,0 +1,5 @@ +# bind and unbind refuse to run while deployment history recording is on. +EnvMatrix.DMS = [""] + +# The case asserts plan actions and skip reasons, not request shapes. +RecordRequests = false diff --git a/acceptance/bundle/resources/materialization_sets/destination/databricks.yml b/acceptance/bundle/resources/materialization_sets/destination/databricks.yml new file mode 100644 index 00000000000..b71b583e8c9 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/databricks.yml @@ -0,0 +1,24 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.destfeat + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: main.myschema.destfeat + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/destination/out.test.toml b/acceptance/bundle/resources/materialization_sets/destination/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/destination/output.txt b/acceptance/bundle/resources/materialization_sets/destination/output.txt new file mode 100644 index 00000000000..dae6c56dd9a --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/output.txt @@ -0,0 +1,101 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A destination the read never returns is not drift: the config holds a value the server +=== never echoes, and with no persisted value to compare it to the suppression lives in +=== OverrideChangeDesc +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "skip", + "store_config_changes": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} + +=== Repointing the online destination is a real change the config asked for, so it keeps its +=== persisted value and still recreates +>>> update_file.py databricks.yml catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on catalog_name: other + schema_name: myschema + table_name_prefix: destfeat_on + +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "recreate", + "store_config_changes": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} + +>>> update_file.py databricks.yml catalog_name: other catalog_name: main + +=== The offline block is input-only whole, and repointing it recreates the same way +>>> update_file.py databricks.yml table_name_prefix: destfeat_off table_name_prefix: destfeat_moved + +>>> [CLI] bundle plan -o json + +>>> destination_changes +{ + "action": "recreate", + "store_config_changes": { + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|is_streaming_mode=false'].offline_store_config.table_name_prefix": { + "action": "recreate", + "reason": "spec:immutable" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.catalog_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.schema_name": { + "action": "skip", + "reason": "input_only_not_returned" + }, + "materialized_features[member='feature_name=main.myschema.destfeat|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix": { + "action": "skip", + "reason": "input_only_not_returned" + } + } +} diff --git a/acceptance/bundle/resources/materialization_sets/destination/script b/acceptance/bundle/resources/materialization_sets/destination/script new file mode 100644 index 00000000000..dde15565eb2 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/destination/script @@ -0,0 +1,32 @@ +trace $CLI bundle deploy + +destination_changes() { + jq '.plan["resources.materialization_sets.set1"] + | {action, store_config_changes: (.changes + | with_entries(select(.key | test("store_config"))) + | map_values({action, reason}))}' out.plan.json +} + +title "A destination the read never returns is not drift: the config holds a value the server" +title "never echoes, and with no persisted value to compare it to the suppression lives in" +title "OverrideChangeDesc" +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes + +title "Repointing the online destination is a real change the config asked for, so it keeps its" +title "persisted value and still recreates" +trace update_file.py databricks.yml ' catalog_name: main + schema_name: myschema + table_name_prefix: destfeat_on' ' catalog_name: other + schema_name: myschema + table_name_prefix: destfeat_on' +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes +trace update_file.py databricks.yml ' catalog_name: other' ' catalog_name: main' + +title "The offline block is input-only whole, and repointing it recreates the same way" +trace update_file.py databricks.yml 'table_name_prefix: destfeat_off' 'table_name_prefix: destfeat_moved' +trace $CLI bundle plan -o json > out.plan.json +trace destination_changes + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl new file mode 100644 index 00000000000..36ae6d6b313 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/added-member.yml.tmpl @@ -0,0 +1,59 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl new file mode 100644 index 00000000000..9be8c5add3c --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/databricks.yml.tmpl @@ -0,0 +1,53 @@ +bundle: + name: test-matset-immutable-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.imma_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # Declared so the set can try to add a member for it. A member names a feature that must + # already exist, and the add is refused by the set update rather than by the feature lookup. + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.immb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: imm_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/immutable/output.txt b/acceptance/bundle/resources/materialization_sets/immutable/output.txt new file mode 100644 index 00000000000..21ed2a4d4e4 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/output.txt @@ -0,0 +1,77 @@ + +>>> create_table.py main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Created table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Inserted sample data into main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A member field the endpoint cannot alter is caught by the planner as a recreate, so the +=== request never reaches the server +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "recreate" +} + +=== Adding a member is routed to update, where the server refuses it by name rather than +=== recreating the set and destroying the members the config left alone +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features cannot be added to an existing materialization set, and feature 'main.materialization_set_acceptance_test.immb_[UNIQUE_NAME]' does not match any member of this set. Recreate the set instead. + +Files: 2 uploaded, 0 deleted + +=== Every refusal above leaves destroy and redeploy as the only route, so that pair is what a +=== member-field change costs. The delete settles its own operation, and the re-created set +=== plans clean. +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.feata + delete resources.features.featb + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default + +Destroy: 3 deleted + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "create" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-immutable-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "skip" +} diff --git a/acceptance/bundle/resources/materialization_sets/immutable/script b/acceptance/bundle/resources/materialization_sets/immutable/script new file mode 100644 index 00000000000..99138f35364 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/script @@ -0,0 +1,52 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-materialization_set_acceptance_test}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the fixture a phase deploys is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +title "A member field the endpoint cannot alter is caught by the planner as a recreate, so the" +title "request never reaches the server" +sed -i 's/0 0 \* \* \* ?/0 30 * * * ?/' databricks.yml +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "Adding a member is routed to update, where the server refuses it by name rather than" +title "recreating the set and destroying the members the config left alone" +render added-member.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Every refusal above leaves destroy and redeploy as the only route, so that pair is what a" +title "member-field change costs. The delete settles its own operation, and the re-created set" +title "plans clean." +render databricks.yml.tmpl +trace $CLI bundle destroy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/immutable/test.toml b/acceptance/bundle/resources/materialization_sets/immutable/test.toml new file mode 100644 index 00000000000..5ad977e46d0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/immutable/test.toml @@ -0,0 +1,33 @@ +# Pinned to the acceptance schema the materialization-set cases share. Override MATSET_CATALOG / +# MATSET_SCHEMA to point it elsewhere. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl new file mode 100644 index 00000000000..340901165da --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/databricks.yml.tmpl @@ -0,0 +1,78 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Both features are materialized to both stores, so either member of a pair cascades to the other. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.feata.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl new file mode 100644 index 00000000000..d8d5e2f7c27 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-feata.yml.tmpl @@ -0,0 +1,61 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's whole pair dropped; featb's pair survives. + materialized_features: + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl new file mode 100644 index 00000000000..4bee71b4545 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-offline.yml.tmpl @@ -0,0 +1,70 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's offline member dropped on its own. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl b/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl new file mode 100644 index 00000000000..5895db5f4cf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/no-online.yml.tmpl @@ -0,0 +1,69 @@ +bundle: + name: test-matset-membership-$UNIQUE_NAME + +resources: + features: + feata: + full_name: $CATALOG_NAME.$SCHEMA_NAME.feata_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featb: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featb_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # feata's online member dropped on its own. + materialized_features: + - feature_name: ${resources.features.feata.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: ${PREFIX}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featb.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/membership/out.test.toml b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/membership/output.txt b/acceptance/bundle/resources/materialization_sets/membership/output.txt new file mode 100644 index 00000000000..35a4b1df921 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/output.txt @@ -0,0 +1,66 @@ + +>>> create_table.py main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Created table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Inserted sample data into main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Created features.feata +Created features.featb +Created materialization_sets.set1 +Files: 5 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== The cascade is per feature and runs both ways. Removing feata's online member alone is +=== refused, because the delete also reaches the offline member the config still declares +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', which the request does not supply. Remove those as well. + +Files: 1 uploaded, 0 deleted + +=== And removing feata's offline member alone is refused for the same reason, from the other +=== direction +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'main.materialization_set_acceptance_test.feata_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 1 uploaded, 0 deleted + +=== Removing a whole feature's pair plans as update, not recreate, and the other feature's +=== members keep their ids, so the set was reconciled rather than rebuilt +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-membership-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> set_members +[ + { + "feature_name": "${resources.features.featb.full_name}", + "is_online": null + }, + { + "feature_name": "${resources.features.featb.full_name}", + "is_online": null + } +] diff --git a/acceptance/bundle/resources/materialization_sets/membership/script b/acceptance/bundle/resources/materialization_sets/membership/script new file mode 100644 index 00000000000..b684f53cc36 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/script @@ -0,0 +1,52 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-materialization_set_acceptance_test}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" +export PREFIX="mat_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the membership a phase declares is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +title "The cascade is per feature and runs both ways. Removing feata's online member alone is" +title "refused, because the delete also reaches the offline member the config still declares" +render no-online.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + +title "And removing feata's offline member alone is refused for the same reason, from the other" +title "direction" +render no-offline.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + +title "Removing a whole feature's pair plans as update, not recreate, and the other feature's" +title "members keep their ids, so the set was reconciled rather than rebuilt" +render no-feata.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace set_members + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/membership/test.toml b/acceptance/bundle/resources/materialization_sets/membership/test.toml new file mode 100644 index 00000000000..d50a94d5701 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/membership/test.toml @@ -0,0 +1,33 @@ +# Pinned to the acceptance schema the materialization-set cases share. Override MATSET_CATALOG / +# MATSET_SCHEMA / MATSET_ONLINE_STORE to point it elsewhere. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl new file mode 100644 index 00000000000..6d2acd4fe7f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/databricks.yml.tmpl @@ -0,0 +1,60 @@ +bundle: + name: test-matset-removal-scope-$UNIQUE_NAME + +resources: + features: + featf1: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf1_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featf2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf2_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Each feature is materialized offline only, so each contributes exactly one member. + materialized_features: + - feature_name: ${resources.features.featf1.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.featf2.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl b/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl new file mode 100644 index 00000000000..ac6257da73e --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/no-f1.yml.tmpl @@ -0,0 +1,52 @@ +bundle: + name: test-matset-removal-scope-$UNIQUE_NAME + +resources: + features: + featf1: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf1_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + featf2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.featf2_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # featf1's only member dropped; featf2 is untouched. + materialized_features: + - feature_name: ${resources.features.featf2.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: $PREFIX + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt new file mode 100644 index 00000000000..22df5d4e500 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/output.txt @@ -0,0 +1,70 @@ + +>>> create_table.py main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Created table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Inserted sample data into main.materialization_set_acceptance_test.src_[UNIQUE_NAME] +Table main.materialization_set_acceptance_test.src_[UNIQUE_NAME] is now visible (catalog_name=main) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-removal-scope-[UNIQUE_NAME]/default/files... +Created features.featf1 +Created features.featf2 +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A feature contributing one member can have it removed alone: the cascade set is that one +=== member, and the feature the request says nothing about is untouched +>>> [CLI] bundle plan -o json + +>>> set_action +{ + "resources.materialization_sets.set1": "update" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-removal-scope-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> set_members +[ + { + "feature_name": "${resources.features.featf2.full_name}", + "is_online": null + } +] + +=== The update returns a long-running operation, and the reconciled set is read back from it +>>> print_requests.py --keep //materialization-sets +{ + "method": "PATCH", + "path": "/api/2.0/feature-engineering/materialization-sets/[UUID]", + "q": { + "request_id": "[UUID]", + "update_mask": "materialized_features" + }, + "body": { + "materialization_set_id": "[UUID]", + "materialized_features": [ + { + "cron_schedule_trigger": { + "cron_expression": "0 0 * * * ?", + "mode": "MANUAL" + }, + "feature_name": "main.materialization_set_acceptance_test.featf2_[UNIQUE_NAME]", + "offline_store_config": { + "catalog_name": "main", + "schema_name": "materialization_set_acceptance_test", + "table_name_prefix": "mat_[UNIQUE_NAME]" + } + } + ] + } +} + +>>> print_requests.py --get //feature-engineering/operations +{ + "method": "GET", + "path": "/api/2.0/feature-engineering/operations/[UUID]" +} diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/script b/acceptance/bundle/resources/materialization_sets/removal_scope/script new file mode 100644 index 00000000000..e0bd2503606 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/script @@ -0,0 +1,49 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-materialization_set_acceptance_test}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" +export PREFIX="mat_$UNIQUE_NAME" + +# Each phase is its own template rather than a string edit, so the membership a phase declares is +# readable on its own. +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +} + +set_members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({feature_name, is_online})' +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +trace create_table.py "$TABLE_NAME" +render databricks.yml.tmpl + +trace $CLI bundle deploy + +# The create's requests are pinned by the basic case, so drop them and let the update's stand alone. +rm -f "$OUT_REQUESTS" + +title "A feature contributing one member can have it removed alone: the cascade set is that one" +title "member, and the feature the request says nothing about is untouched" +render no-f1.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace set_members + +title "The update returns a long-running operation, and the reconciled set is read back from it" +trace print_requests.py --keep //materialization-sets +trace print_requests.py --get //feature-engineering/operations + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml new file mode 100644 index 00000000000..d50a94d5701 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/removal_scope/test.toml @@ -0,0 +1,33 @@ +# Pinned to the acceptance schema the materialization-set cases share. Override MATSET_CATALOG / +# MATSET_SCHEMA / MATSET_ONLINE_STORE to point it elsewhere. +Cloud = true + +# Both legs run locally. A cloud run must select the DMS= leg: the bundle deployments API +# rejects features. and materialization_sets. as resource-type prefixes. +EnvMatrix.DMS = ["", "true"] + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# Fake SQL endpoint so create_table.py works against the local test server too. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +# The cleanup drops the source table; the fake has no delete handler for one. +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl new file mode 100644 index 00000000000..8f76bfee764 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/databricks.yml.tmpl @@ -0,0 +1,51 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl new file mode 100644 index 00000000000..34db72c23f8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-all.yml.tmpl @@ -0,0 +1,25 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: [] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl new file mode 100644 index 00000000000..cfae25e8a3b --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-batch-pair.yml.tmpl @@ -0,0 +1,34 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl new file mode 100644 index 00000000000..9f86ac53015 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-offline.yml.tmpl @@ -0,0 +1,43 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl new file mode 100644 index 00000000000..3969ab6e672 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/drop-online-batch.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online streaming. RTM is the only streaming mode a non-Kinesis stream source accepts. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt new file mode 100644 index 00000000000..1cb3dd4ed82 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/output.txt @@ -0,0 +1,117 @@ + +=== A sawtooth feature declaring only the offline and online batch pair is refused at create. +=== The pair is not expanded into the third materialization: a set stamps its id on every +=== member before the backend reaches that normalization, and the normalization is skipped +=== whenever the id is set, so one member stays one materialized feature. +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Created features.sawfeat +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. (200 BAD_REQUEST) + +Endpoint: n/a +HTTP Status: 200 +API error_code: BAD_REQUEST +API message: materialization set [UUID] operation failed: Feature 'danielb.fs2680.saw_[UNIQUE_NAME]' uses a sawtooth window, which requires an offline batch materialization, an online batch materialization and an online streaming materialization. Declare one materialize feature request for each. + +Files: 7 uploaded, 0 deleted + +=== Declaring all three creates the set +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +=== A redeploy plans nothing. Both online members carry a destination the read never returns, +=== so this is also what proves the input-only suppression holds for a three-member set. +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 2 unchanged + +=== Every suppressed destination change, by reason +>>> jq -r .plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length out.plan.json +7 + +=== A sawtooth feature's three members are one cascade group, so every strict subset of them +=== is refused. Removing the offline member reaches both online ones, +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== removing the online batch member names no member a delete can be applied to, +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request does not supply. Remove those as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Materialized features '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' are deleted only by deleting materialized features '[UUID]', '[UUID]', which the request does not supply. Remove those as well. + +Files: 2 uploaded, 0 deleted + +=== and removing the batch pair still leaves the streaming member behind +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Deleting materialized features '[UUID]', '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Deleting materialized features '[UUID]', '[UUID]' of feature 'danielb.fs2680.saw_[UNIQUE_NAME]' also deletes materialized features '[UUID]', which the request does not delete. Delete the unrequested materialized features as well. + +Files: 2 uploaded, 0 deleted + +=== Removing all three satisfies the cascade but empties the set, which the update endpoint +=== refuses in favour of the delete endpoint. None of the four refusals wrote anything, so +=== restoring the config plans nothing. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-sawtooth-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. (400 INVALID_PARAMETER_VALUE) + +Endpoint: PATCH [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets/[UUID]?request_id=[UUID]&update_mask=materialized_features +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: Updating materialization set '[UUID]' with no materialized features is not supported. Delete the set instead. + +Files: 2 uploaded, 0 deleted + +>>> [CLI] bundle plan -o json + +>>> set_action +skip diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/script b/acceptance/bundle/resources/materialization_sets/sawtooth/script new file mode 100644 index 00000000000..0184d2f2f84 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/script @@ -0,0 +1,82 @@ +# A sawtooth feature needs a Stream source, and a Stream is not a bundle resource, so this case +# references one that already exists rather than creating it. Everything else is per-run. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" + +# The harness resolves auth from the ambient environment, which on a cloud run is the workspace +# under test. Pointing a local run at a named profile needs it in the bundle, because +# DATABRICKS_CONFIG_PROFILE does not survive into the CLI subprocess. +pin_profile() { + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +render() { + envsubst < "$1" > databricks.yml + pin_profile +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true +} +trap cleanup EXIT + +title "A sawtooth feature declaring only the offline and online batch pair is refused at create." +title "The pair is not expanded into the third materialization: a set stamps its id on every" +title "member before the backend reaches that normalization, and the normalization is skipped" +title "whenever the id is set, so one member stays one materialized feature." +envsubst < twoleg.yml.tmpl > databricks.yml +pin_profile +trace musterr $CLI bundle deploy + +title "Declaring all three creates the set" +envsubst < databricks.yml.tmpl > databricks.yml +pin_profile +trace $CLI bundle deploy + +title "A redeploy plans nothing. Both online members carry a destination the read never returns," +title "so this is also what proves the input-only suppression holds for a three-member set." +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy + +title "Every suppressed destination change, by reason" +trace jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries | map(select(.value.reason == "input_only_not_returned")) | length' out.plan.json + +title "A sawtooth feature's three members are one cascade group, so every strict subset of them" +title "is refused. Removing the offline member reaches both online ones," +render drop-offline.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "removing the online batch member names no member a delete can be applied to," +render drop-online-batch.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "and removing the batch pair still leaves the streaming member behind" +render drop-batch-pair.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Removing all three satisfies the cascade but empties the set, which the update endpoint" +title "refuses in favour of the delete endpoint. None of the four refusals wrote anything, so" +title "restoring the config plans nothing." +render drop-all.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve +render databricks.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml new file mode 100644 index 00000000000..92c3ef0e316 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/test.toml @@ -0,0 +1,19 @@ +# Pinned to a workspace holding a Stream, since a Stream is not a bundle resource. Override +# MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' diff --git a/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl b/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl new file mode 100644 index 00000000000..54b4be4fb96 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/sawtooth/twoleg.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-sawtooth-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.saw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 259200s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Offline batch, on the schedule the service derives for a sawtooth window. + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: DERIVED + # Online batch, sharing its destination with the streaming member below. + - feature_name: ${resources.features.sawfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: saw_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml new file mode 100644 index 00000000000..d4064e0b26a --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/databricks.yml @@ -0,0 +1,19 @@ +bundle: + name: test-bundle + +resources: + materialization_sets: + set1: + materialized_features: + - feature_name: main.myschema.featA + offline_store_config: + catalog_name: main + schema_name: myschema + table_name_prefix: offline + pipeline_schedule_state: ACTIVE + - feature_name: main.myschema.featA + online_store_config: + online_store_name: danielb + catalog_name: main + schema_name: myschema + table_name_prefix: feata diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml b/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml new file mode 100644 index 00000000000..a927a5fbc06 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt new file mode 100644 index 00000000000..cade39560de --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/output.txt @@ -0,0 +1,86 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 0 unchanged + +=== A member the config leaves unset adopts the state the server set on create, so the next +=== plan is empty rather than reporting drift the bundle cannot write +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle summary -o json +[ + { + "is_online": null, + "pipeline_schedule_state": "ACTIVE" + }, + { + "is_online": null, + "pipeline_schedule_state": null + } +] + +=== The state is write-once, so pausing the feature is a recreate rather than the in-place +=== update the server would refuse. Every member of the feature has to ask for the same state +>>> update_file.py databricks.yml pipeline_schedule_state: ACTIVE pipeline_schedule_state: PAUSED + +>>> update_file.py databricks.yml table_name_prefix: feata table_name_prefix: feata + pipeline_schedule_state: PAUSED + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "recreate" +} + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 0 unchanged + +>>> [CLI] bundle summary -o json +[ + { + "is_online": null, + "pipeline_schedule_state": "PAUSED" + }, + { + "is_online": null, + "pipeline_schedule_state": "PAUSED" + } +] + +=== Resuming one member alone plans as a recreate too, and the create endpoint refuses the +=== members of one feature disagreeing +>>> update_file.py databricks.yml pipeline_schedule_state: PAUSED + - feature_name pipeline_schedule_state: ACTIVE + - feature_name + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-bundle/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Error: cannot recreate resources.materialization_sets.set1: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: All materialized features in a BatchCreateMaterializedFeaturesRequest call must have the same pipeline schedule state. + +Files: 1 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/schedule_state/script b/acceptance/bundle/resources/materialization_sets/schedule_state/script new file mode 100644 index 00000000000..6c65ff96bbf --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/schedule_state/script @@ -0,0 +1,26 @@ +trace $CLI bundle deploy + +title "A member the config leaves unset adopts the state the server set on create, so the next" +title "plan is empty rather than reporting drift the bundle cannot write" +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' + +title "The state is write-once, so pausing the feature is a recreate rather than the in-place" +title "update the server would refuse. Every member of the feature has to ask for the same state" +trace update_file.py databricks.yml 'pipeline_schedule_state: ACTIVE' 'pipeline_schedule_state: PAUSED' +trace update_file.py databricks.yml ' table_name_prefix: feata' ' table_name_prefix: feata + pipeline_schedule_state: PAUSED' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy --auto-approve +trace $CLI bundle summary -o json | jq '.resources.materialization_sets.set1.materialized_features | map({is_online, pipeline_schedule_state})' + +title "Resuming one member alone plans as a recreate too, and the create endpoint refuses the" +title "members of one feature disagreeing" +trace update_file.py databricks.yml ' pipeline_schedule_state: PAUSED + - feature_name' ' pipeline_schedule_state: ACTIVE + - feature_name' +trace musterr $CLI bundle deploy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl new file mode 100644 index 00000000000..217618ecf63 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/databricks.yml.tmpl @@ -0,0 +1,79 @@ +bundle: + name: test-matset-settled-$UNIQUE_NAME + +resources: + features: + okfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.ok_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # The fake refuses to tear down any member of a feature whose name carries this marker, which + # is how a fixture asks for a failure that real infrastructure would produce. + failfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.teardown_fails_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: id + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.okfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.okfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.failfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.failfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl b/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl new file mode 100644 index 00000000000..ee6c39a3883 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/drop-failing.yml.tmpl @@ -0,0 +1,62 @@ +bundle: + name: test-matset-settled-$UNIQUE_NAME + +resources: + features: + okfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.ok_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + # The fake refuses to tear down any member of a feature whose name carries this marker, which + # is how a fixture asks for a failure that real infrastructure would produce. + failfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.teardown_fails_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: id + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + materialized_features: + - feature_name: ${resources.features.okfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.okfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: sf_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml b/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml new file mode 100644 index 00000000000..19b5b15d11f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/out.test.toml @@ -0,0 +1,2 @@ +Cloud = false +EnvMatrix.DMS = [""] diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt new file mode 100644 index 00000000000..38912722d51 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/output.txt @@ -0,0 +1,48 @@ + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... +Created features.failfeat +Created features.okfeat +Created materialization_sets.set1 +Files: 3 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A member the server cannot tear down settles the update's operation as FAILED and returns +=== it with HTTP 200. The adapter reads the code off the operation and the per-member outcome +=== off its metadata, so the deploy fails naming each member left behind. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default/files... +Error: cannot update resources.materialization_sets.set1: updating id=[UUID]: materialization set [UUID] operation failed: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. (200 INTERNAL_ERROR) + +Endpoint: n/a +HTTP Status: 200 +API error_code: INTERNAL_ERROR +API message: materialization set [UUID] operation failed: Changed 0 of 2 materialized features in materialization set '[UUID]'. The operation's progress lists which update succeeded, which failed and why.; members left unchanged: [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support.; [UUID] (INTERNAL_ERROR): Failed to delete materialized feature '[UUID]'. If the issue persists, please retry or contact support. + +Files: 2 uploaded, 0 deleted + +=== Nothing was written, so the removal is still pending +>>> [CLI] bundle plan -o json + +>>> set_action +update + +=== The delete settles FAILED the same way, and here the engine decides what the user sees: +=== its destroy step forgives a delete error matching none of the cases it knows, so the +=== destroy reports success and drops the state while the set stays in the workspace. The leg +=== with deployment-history recording takes the one branch that records and returns the +=== failure, which is why this case runs on the other leg only. +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.failfeat + delete resources.features.okfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-settled-[UNIQUE_NAME]/default + +Destroy: 3 deleted diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/script b/acceptance/bundle/resources/materialization_sets/settled_failure/script new file mode 100644 index 00000000000..b0882e8c2ed --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/script @@ -0,0 +1,36 @@ +export CATALOG_NAME="${MATSET_CATALOG:-main}" +export SCHEMA_NAME="${MATSET_SCHEMA:-myschema}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +render() { + envsubst < "$1" > databricks.yml +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +render databricks.yml.tmpl +trace $CLI bundle deploy + +title "A member the server cannot tear down settles the update's operation as FAILED and returns" +title "it with HTTP 200. The adapter reads the code off the operation and the per-member outcome" +title "off its metadata, so the deploy fails naming each member left behind." +render drop-failing.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace musterr $CLI bundle deploy --auto-approve + +title "Nothing was written, so the removal is still pending" +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "The delete settles FAILED the same way, and here the engine decides what the user sees:" +title "its destroy step forgives a delete error matching none of the cases it knows, so the" +title "destroy reports success and drops the state while the set stays in the workspace. The leg" +title "with deployment-history recording takes the one branch that records and returns the" +title "failure, which is why this case runs on the other leg only." +trace $CLI bundle destroy --auto-approve + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml b/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml new file mode 100644 index 00000000000..61fdc6235a0 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/settled_failure/test.toml @@ -0,0 +1,12 @@ +# Pinned to the leg without deployment-history recording, where the engine's destroy step forgives +# a delete error it cannot classify. With recording on it takes the branch that records the failure +# and returns it, so the destroy fails instead and the two legs cannot share one golden. +EnvMatrix.DMS = [""] + +# Local only, and not because the claims are redundant: the failure is produced by a marker the +# fake looks for in a feature name, since a real teardown fails on infrastructure no request can +# ask for. Against a workspace this case would simply succeed. +Ignore = [ + ".databricks", + "databricks.yml", +] diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl new file mode 100644 index 00000000000..0b60914a07f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/batch-only.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + pairfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.pair_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # The shorthand is the whole declaration: every member of this set was produced by expanding + # the one entry below. + batch_offline_and_online_materialized_features: + - feature_name: ${resources.features.pairfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl new file mode 100644 index 00000000000..a0ce3b72f32 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/batch-pair.yml.tmpl @@ -0,0 +1,77 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + pairfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.pair_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 2d1h35m40s + slide_duration: 1h30m + entities: + - name: id + timeseries_column: + name: timestamp + otherfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.other_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 2d1h35m40s + slide_duration: 1h30m + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # pairfeat is declared once and expanded into its offline and online members; otherfeat is + # written out in full. Every feature in a set must ask for the same destination pair, so the + # two declarations name identical destinations. + batch_offline_and_online_materialized_features: + - feature_name: ${resources.features.pairfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + materialized_features: + - feature_name: ${resources.features.otherfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_${UNIQUE_NAME}_off + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + - feature_name: ${resources.features.otherfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: pair_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl new file mode 100644 index 00000000000..070f265c555 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/both-shorthand.yml.tmpl @@ -0,0 +1,72 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + bothsaw: + full_name: $CATALOG_NAME.$SCHEMA_NAME.bothsaw_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 3d + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + bothbatch: + full_name: $CATALOG_NAME.$SCHEMA_NAME.bothbatch_$UNIQUE_NAME + source: + delta_table_source: + full_name: $TABLE_NAME + function: + aggregation_function: + count_function: + input: value + time_window: + sliding: + window_duration: 604800s + slide_duration: 86400s + entities: + - name: id + timeseries_column: + name: timestamp + + materialization_sets: + set1: + # Both shorthand fields at once. The two entries name identical destinations, so what the + # service refuses is the pair of cron schedule modes the two expansions ask for. + sawtooth_materialized_features: + - feature_name: ${resources.features.bothsaw.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + batch_offline_and_online_materialized_features: + - feature_name: ${resources.features.bothbatch.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: both_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl new file mode 100644 index 00000000000..3577aa6f0d1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/databricks.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.shd_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 3d + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + # One entry for the whole of this feature's materialization. Both destinations and both + # triggers are written once; the CLI splits them across the three members the service wants. + sawtooth_materialized_features: + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_$UNIQUE_NAME + cron_schedule_trigger: + mode: DERIVED + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl new file mode 100644 index 00000000000..e1947c6e714 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/manual-schedule.yml.tmpl @@ -0,0 +1,42 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.shd_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 3d + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + # One entry for the whole of this feature's materialization. Both destinations and both + # triggers are written once; the CLI splits them across the three members the service wants. + sawtooth_materialized_features: + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_$UNIQUE_NAME + cron_schedule_trigger: + mode: MANUAL + cron_expression: "0 0 * * * ?" + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl new file mode 100644 index 00000000000..dbd5ed03cf8 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/moved.yml.tmpl @@ -0,0 +1,41 @@ +bundle: + name: test-matset-shorthand-$UNIQUE_NAME + +resources: + features: + sawfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.shd_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + sum: + input: value.passenger_count_1 + time_window: + sawtooth: + window_duration: 3d + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + # One entry for the whole of this feature's materialization. Both destinations and both + # triggers are written once; the CLI splits them across the three members the service wants. + sawtooth_materialized_features: + - feature_name: ${resources.features.sawfeat.full_name} + offline_store_config: + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_off + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: shd_${UNIQUE_NAME}_moved + cron_schedule_trigger: + mode: DERIVED + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml b/acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/output.txt b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt new file mode 100644 index 00000000000..66ed29c2d3f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/output.txt @@ -0,0 +1,191 @@ + +=== The shorthand judges nothing it is given. A batch schedule the service refuses is reported +=== by the service, naming the feature, rather than rejected while the config is being read. +>>> musterr [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.sawfeat +Error: cannot create resources.materialization_sets.set1: materialization set [UUID] operation failed: Feature 'danielb.fs2680.shd_[UNIQUE_NAME]' must declare a DERIVED cron schedule on its batch materialize feature requests. The sawtooth batch materializations refresh on a daily schedule the service computes, so a caller neither pins nor omits it. (200 BAD_REQUEST) + +Endpoint: n/a +HTTP Status: 200 +API error_code: BAD_REQUEST +API message: materialization set [UUID] operation failed: Feature 'danielb.fs2680.shd_[UNIQUE_NAME]' must declare a DERIVED cron schedule on its batch materialize feature requests. The sawtooth batch materializations refresh on a daily schedule the service computes, so a caller neither pins nor omits it. + +Files: 7 uploaded, 0 deleted + +=== Declaring the same feature once with a DERIVED batch schedule creates the three members it +=== is made of, and neither shorthand field survives into the summary +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created materialization_sets.set1 +Files: 1 uploaded, 0 deleted +Resources: 1 created, 0 changed, 0 deleted, 1 unchanged + +>>> members +{ + "sawtooth_shorthand_remains": false, + "batch_pair_shorthand_remains": false, + "members": [ + { + "feature": "sawfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "sawfeat", + "destination": "online", + "trigger": "batch" + }, + { + "feature": "sawfeat", + "destination": "online", + "trigger": "streaming" + } + ] +} + +=== A redeploy plans nothing, so the expansion is the same on every invocation +>>> [CLI] bundle plan -o json + +>>> set_action +skip + +=== The online destination is written once, so one edit reaches both online members. The offline +=== member, whose destination did not change, is absent from the changes. +>>> [CLI] bundle plan -o json + +>>> set_action +recreate + +>>> recreated_fields +materialized_features[member='feature_name=danielb.fs2680.shd_[UNIQUE_NAME]|online_store_name=danielb|is_streaming_mode=false'].online_store_config.table_name_prefix +materialized_features[member='feature_name=danielb.fs2680.shd_[UNIQUE_NAME]|online_store_name=danielb|is_streaming_mode=true'].online_store_config.table_name_prefix + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... + +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change: + recreate resources.materialization_sets.set1 +Recreated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 1 created, 0 changed, 1 deleted, 1 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +skip + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.sawfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default + +Destroy: 2 deleted + +=== A set may mix a feature declared in shorthand with members written out in full: the +=== expansion is appended to what the config already lists +>>> create_table.py danielb.fs2680.src_[UNIQUE_NAME] +Created table danielb.fs2680.src_[UNIQUE_NAME] +Inserted sample data into danielb.fs2680.src_[UNIQUE_NAME] +Table danielb.fs2680.src_[UNIQUE_NAME] is now visible (catalog_name=danielb) + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.otherfeat +Created features.pairfeat +Created materialization_sets.set1 +Files: 8 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +>>> members +{ + "sawtooth_shorthand_remains": false, + "batch_pair_shorthand_remains": false, + "members": [ + { + "feature": "otherfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "otherfeat", + "destination": "online", + "trigger": "batch" + }, + { + "feature": "pairfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "pairfeat", + "destination": "online", + "trigger": "batch" + } + ] +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.otherfeat + delete resources.features.pairfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default + +Destroy: 3 deleted + +=== The batch pair shorthand carries a whole set on its own, with no member written out +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.pairfeat +Created materialization_sets.set1 +Files: 8 uploaded, 0 deleted +Resources: 2 created, 0 changed, 0 deleted, 0 unchanged + +>>> members +{ + "sawtooth_shorthand_remains": false, + "batch_pair_shorthand_remains": false, + "members": [ + { + "feature": "pairfeat", + "destination": "offline", + "trigger": "batch" + }, + { + "feature": "pairfeat", + "destination": "online", + "trigger": "batch" + } + ] +} + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.features.pairfeat + delete resources.materialization_sets.set1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default + +Destroy: 2 deleted + +=== Both shorthand fields expand in the same set, which the service then refuses: a sawtooth +=== entry has to carry a DERIVED schedule and a batch pair a MANUAL one, and one request cannot +=== mix the two modes. The CLI expanded both fields and judged neither. +>>> musterr [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-shorthand-[UNIQUE_NAME]/default/files... +Created features.bothbatch +Created features.bothsaw +Error: cannot create resources.materialization_sets.set1: All cron-scheduled features in a single request must use the same cron schedule mode; a request cannot mix DERIVED (service-derived) and MANUAL (caller-supplied) schedules. (400 INVALID_PARAMETER_VALUE) + +Endpoint: POST [DATABRICKS_URL]/api/2.0/feature-engineering/materialization-sets?request_id=[UUID] +HTTP Status: 400 Bad Request +API error_code: INVALID_PARAMETER_VALUE +API message: All cron-scheduled features in a single request must use the same cron schedule mode; a request cannot mix DERIVED (service-derived) and MANUAL (caller-supplied) schedules. + +Files: 8 uploaded, 0 deleted diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/script b/acceptance/bundle/resources/materialization_sets/shorthand/script new file mode 100644 index 00000000000..b0f0f4284d4 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/script @@ -0,0 +1,94 @@ +# The sawtooth half needs a Stream source, and a Stream is not a bundle resource, so it references +# one that already exists. The batch half creates its own Delta source. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" +export TABLE_NAME="$CATALOG_NAME.$SCHEMA_NAME.src_$UNIQUE_NAME" + +render() { + envsubst < "$1" > databricks.yml + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +# The shorthand fields must not survive the expansion, so the summary is the whole picture of what +# was declared. +members() { + $CLI bundle summary -o json | jq '.resources.materialization_sets.set1 | { + sawtooth_shorthand_remains: has("sawtooth_materialized_features"), + batch_pair_shorthand_remains: has("batch_offline_and_online_materialized_features"), + members: [.materialized_features[] | { + feature: (.feature_name | ltrimstr("${resources.features.") | rtrimstr(".full_name}")), + destination: (if .online_store_config != null then "online" else "offline" end), + trigger: (if .streaming_mode != null then "streaming" else "batch" end) + }] + }' +} + +# Only the fields that force the recreate, so the claim is which members the one edit reached. +recreated_fields() { + jq -r '.plan["resources.materialization_sets.set1"].changes | to_entries + | map(select(.value.action == "recreate")) | map(.key)[]' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true + $CLI tables delete "$TABLE_NAME" > /dev/null 2>&1 || true +} +trap cleanup EXIT + +title "The shorthand judges nothing it is given. A batch schedule the service refuses is reported" +title "by the service, naming the feature, rather than rejected while the config is being read." +render manual-schedule.yml.tmpl +trace musterr $CLI bundle deploy + +title "Declaring the same feature once with a DERIVED batch schedule creates the three members it" +title "is made of, and neither shorthand field survives into the summary" +render databricks.yml.tmpl +trace $CLI bundle deploy +trace members + +title "A redeploy plans nothing, so the expansion is the same on every invocation" +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "The online destination is written once, so one edit reaches both online members. The offline" +title "member, whose destination did not change, is absent from the changes." +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace recreated_fields +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +trace $CLI bundle destroy --auto-approve + +title "A set may mix a feature declared in shorthand with members written out in full: the" +title "expansion is appended to what the config already lists" +trace create_table.py "$TABLE_NAME" +render batch-pair.yml.tmpl +trace $CLI bundle deploy +trace members + +trace $CLI bundle destroy --auto-approve + +title "The batch pair shorthand carries a whole set on its own, with no member written out" +render batch-only.yml.tmpl +trace $CLI bundle deploy +trace members + +trace $CLI bundle destroy --auto-approve + +title "Both shorthand fields expand in the same set, which the service then refuses: a sawtooth" +title "entry has to carry a DERIVED schedule and a batch pair a MANUAL one, and one request cannot" +title "mix the two modes. The CLI expanded both fields and judged neither." +render both-shorthand.yml.tmpl +trace musterr $CLI bundle deploy --auto-approve + +rm -f out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/shorthand/test.toml b/acceptance/bundle/resources/materialization_sets/shorthand/test.toml new file mode 100644 index 00000000000..05d46f50e20 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/shorthand/test.toml @@ -0,0 +1,30 @@ +# Mirrors sawtooth/test.toml: pinned to a workspace holding a Stream, since a Stream is not a +# bundle resource. Override MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM +# to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' + +# The batch half creates and drops a Delta source, so the local run needs the same two stubs +# batch/test.toml carries. +[[Server]] +Pattern = "POST /api/2.0/sql/statements/" +Response.Body = '{"status": {"state": "SUCCEEDED"}, "manifest": {"schema": {"columns": []}}}' + +[[Server]] +Pattern = "DELETE /api/2.1/unity-catalog/tables/{full_name}" +Response.Body = '{}' diff --git a/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl new file mode 100644 index 00000000000..5e09efb43e1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/databricks.yml.tmpl @@ -0,0 +1,64 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat2.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl new file mode 100644 index 00000000000..a425fee6857 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/drop-one.yml.tmpl @@ -0,0 +1,55 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl b/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl new file mode 100644 index 00000000000..005b9a122ed --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/moved.yml.tmpl @@ -0,0 +1,64 @@ +bundle: + name: test-matset-streaming-$UNIQUE_NAME + +resources: + features: + # A second feature materialized to the same destination, so removing one feature's member + # leaves a set behind. A set must be emptied through the delete endpoint, not an update. + streamfeat2: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream2_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + streamfeat: + full_name: $CATALOG_NAME.$SCHEMA_NAME.stream_$UNIQUE_NAME + source: + stream_source: + full_name: $STREAM_FULL_NAME + function: + aggregation_function: + count_function: + input: value.passenger_count_1 + # A Stream source accepts a rolling or a sawtooth window. Rolling keeps this feature on + # the plain streaming path, where one member is one materialized feature. + time_window: + rolling: + window_duration: 3600s + entities: + - name: value.pickup_zip + timeseries_column: + name: value.event_timestamp + + materialization_sets: + set1: + materialized_features: + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_${UNIQUE_NAME}_moved + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM + # Online streaming only: no cron schedule, and no pipeline schedule state to agree on. + - feature_name: ${resources.features.streamfeat2.full_name} + online_store_config: + online_store_name: $ONLINE_STORE_NAME + catalog_name: $CATALOG_NAME + schema_name: $SCHEMA_NAME + table_name_prefix: stream_$UNIQUE_NAME + streaming_mode: + mode: STREAMING_MODE_TYPE_RTM diff --git a/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml b/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml new file mode 100644 index 00000000000..0f167effef1 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/out.test.toml @@ -0,0 +1,2 @@ +Cloud = true +EnvMatrix.DMS = ["", "true"] diff --git a/acceptance/bundle/resources/materialization_sets/streaming/output.txt b/acceptance/bundle/resources/materialization_sets/streaming/output.txt new file mode 100644 index 00000000000..b9ee84b6ae9 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/output.txt @@ -0,0 +1,61 @@ + +=== A streaming member carries a trigger and no schedule, so none of the batch rules apply +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Created features.streamfeat +Created features.streamfeat2 +Created materialization_sets.set1 +Files: 4 uploaded, 0 deleted +Resources: 3 created, 0 changed, 0 deleted, 0 unchanged + +=== A redeploy plans nothing. The schedule state the server assigns to a lone streaming member +=== must not read as drift, and neither must its online destination. +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "skip" +} + +>>> [CLI] bundle deploy +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Files: 1 uploaded, 0 deleted +Resources: 0 created, 0 changed, 0 deleted, 3 unchanged + +=== The schedule state is write-once, so declaring one on a streaming member that had none +=== plans as a recreate rather than an in-place update +>>> update_file.py databricks.yml streaming_mode: pipeline_schedule_state: PAUSED + streaming_mode: + +>>> [CLI] bundle plan -o json + +>>> jq .plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action) out.plan.json +{ + "resources.materialization_sets.set1": "recreate" +} + +=== Repointing a member's online destination is a recreate: the server cannot move a +=== materialization in place, so the whole set is rebuilt at the new location +>>> [CLI] bundle plan -o json + +>>> set_action +recreate + +=== Removing one feature's streaming member is an update the endpoint accepts. With no offline +=== member to cascade from, each online member is its own cascade group, so the other feature's +=== member is untouched and the set is left non-empty. +>>> [CLI] bundle plan -o json + +>>> set_action +update + +>>> [CLI] bundle deploy --auto-approve +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-matset-streaming-[UNIQUE_NAME]/default/files... +Updated materialization_sets.set1 +Files: 2 uploaded, 0 deleted +Resources: 0 created, 1 changed, 0 deleted, 2 unchanged + +>>> [CLI] bundle plan -o json + +>>> set_action +skip diff --git a/acceptance/bundle/resources/materialization_sets/streaming/script b/acceptance/bundle/resources/materialization_sets/streaming/script new file mode 100644 index 00000000000..16a98ab9394 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/script @@ -0,0 +1,64 @@ +# Like the sawtooth case, this references an existing Stream rather than creating one. +export CATALOG_NAME="${MATSET_CATALOG:-danielb}" +export SCHEMA_NAME="${MATSET_SCHEMA:-fs2680}" +export ONLINE_STORE_NAME="${MATSET_ONLINE_STORE:-danielb}" +export STREAM_FULL_NAME="${MATSET_STREAM:-danielb.fs2680.fs2680_dch48o_stream}" + +# The harness resolves auth from the ambient environment, which on a cloud run is the workspace +# under test. Pointing a local run at a named profile needs it in the bundle, because +# DATABRICKS_CONFIG_PROFILE does not survive into the CLI subprocess. +pin_profile() { + [ -n "${MATSET_PROFILE:-}" ] || return 0 + printf "\nworkspace:\n profile: %s\n" "$MATSET_PROFILE" >> databricks.yml +} + +render() { + envsubst < "$1" > databricks.yml + pin_profile +} + +set_action() { + jq -r '.plan["resources.materialization_sets.set1"].action' out.plan.json +} + +cleanup() { + $CLI bundle destroy --auto-approve > /dev/null 2>&1 || true +} +trap cleanup EXIT + +envsubst < databricks.yml.tmpl > databricks.yml +pin_profile + +title "A streaming member carries a trigger and no schedule, so none of the batch rules apply" +trace $CLI bundle deploy + +title "A redeploy plans nothing. The schedule state the server assigns to a lone streaming member" +title "must not read as drift, and neither must its online destination." +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json +trace $CLI bundle deploy + +title "The schedule state is write-once, so declaring one on a streaming member that had none" +title "plans as a recreate rather than an in-place update" +trace update_file.py databricks.yml ' streaming_mode:' ' pipeline_schedule_state: PAUSED + streaming_mode:' +trace $CLI bundle plan -o json > out.plan.json +trace jq '.plan | with_entries(select(.key | startswith("resources.materialization_sets"))) | map_values(.action)' out.plan.json + +title "Repointing a member's online destination is a recreate: the server cannot move a" +title "materialization in place, so the whole set is rebuilt at the new location" +render moved.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +title "Removing one feature's streaming member is an update the endpoint accepts. With no offline" +title "member to cascade from, each online member is its own cascade group, so the other feature's" +title "member is untouched and the set is left non-empty." +render drop-one.yml.tmpl +trace $CLI bundle plan -o json > out.plan.json +trace set_action +trace $CLI bundle deploy --auto-approve +trace $CLI bundle plan -o json > out.plan.json +trace set_action + +rm -f out.requests.txt out.plan.json diff --git a/acceptance/bundle/resources/materialization_sets/streaming/test.toml b/acceptance/bundle/resources/materialization_sets/streaming/test.toml new file mode 100644 index 00000000000..92c3ef0e316 --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/streaming/test.toml @@ -0,0 +1,19 @@ +# Pinned to a workspace holding a Stream, since a Stream is not a bundle resource. Override +# MATSET_CATALOG / MATSET_SCHEMA / MATSET_ONLINE_STORE / MATSET_STREAM to point it elsewhere. +Cloud = true + +Ignore = [ + ".databricks", + "databricks.yml", +] + +# A real materialization set or member id is 32 undashed hex characters; the harness's built-in +# replacement only matches the dashed form the fake produces. +[[Repls]] +Old = '\b[0-9a-f]{32}\b' +New = '[UUID]' + +# The workspace host, which only a cloud run puts in the output. +[[Repls]] +Old = 'https://[a-z0-9.-]+\.cloud\.databricks\.com' +New = '[DATABRICKS_URL]' diff --git a/acceptance/bundle/resources/materialization_sets/test.toml b/acceptance/bundle/resources/materialization_sets/test.toml new file mode 100644 index 00000000000..1b9a24bbf2f --- /dev/null +++ b/acceptance/bundle/resources/materialization_sets/test.toml @@ -0,0 +1,7 @@ +# Local-only for now. The fake in libs/testserver/materialization_set.go mirrors +# UpdateMaterializationSet's reconcile semantics, so these goldens are a claim about the real API; +# turning on Cloud = true checks that claim against a workspace. See README.md for the delta. +Ignore = [ + "databricks.yml", + ".databricks", +] diff --git a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go index 4ead9785e06..3b3eef2fe64 100644 --- a/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go +++ b/bundle/config/mutator/resourcemutator/apply_bundle_permissions_test.go @@ -24,6 +24,7 @@ var unsupportedResources = []string{ "volumes", "schemas", "features", + "materialization_sets", "quality_monitors", "registered_models", "model_services", diff --git a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go index e9ea418d1d6..fd5f61bc111 100644 --- a/bundle/config/mutator/resourcemutator/apply_target_mode_test.go +++ b/bundle/config/mutator/resourcemutator/apply_target_mode_test.go @@ -159,6 +159,13 @@ func mockBundle(mode config.Mode) *bundle.Bundle { Features: map[string]*resources.Feature{ "feature1": {Feature: ml.Feature{FullName: "catalog1.schema1.feature1"}}, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "materialization_set1": {MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "catalog1.schema1.feature1"}, + }, + }}, + }, Clusters: map[string]*resources.Cluster{ "cluster1": {ClusterSpec: compute.ClusterSpec{ClusterName: "cluster1", SparkVersion: "13.2.x", NumWorkers: 1}}, }, diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go index a0e8d05fbde..f2893698bcb 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies.go @@ -14,13 +14,15 @@ import ( type captureUCDependencies struct{} // If a user defines a UC schema in the bundle, they can refer to it in SDP pipelines, -// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, or Vector Search -// Indexes using the `${resources.schemas..name}` syntax. Using this syntax allows +// UC Volumes, Registered Models, Quality Monitors, Model Serving Endpoints, Vector Search +// Indexes, Features, or Materialization Sets using the `${resources.schemas..name}` +// syntax. Using this syntax allows // TF to capture the deploy time dependency this resource has on the schema and deploy changes // to the schema before deploying the dependent resource. // // Similarly, if a user defines a UC catalog in the bundle, they can refer to it in UC schemas, -// UC Volumes, Registered Models, Model Serving Endpoints, or Vector Search Indexes using the +// UC Volumes, Registered Models, Model Serving Endpoints, Vector Search Indexes, Features, or +// Materialization Sets using the // `${resources.catalogs..name}` syntax. This captures the deploy time // dependency the resource has on the catalog. // @@ -204,6 +206,22 @@ func (m *captureUCDependencies) Apply(ctx context.Context, b *bundle.Bundle) dia catalogName, schemaName := parts[0], parts[1] f.FullName = resolveCatalog(b, catalogName) + "." + resolveSchema(b, catalogName, schemaName) + "." + parts[2] } + for _, set := range b.Config.Resources.MaterializationSets { + if set == nil { + continue + } + for i := range set.MaterializedFeatures { + member := &set.MaterializedFeatures[i] + if offline := member.OfflineStoreConfig; offline != nil { + offline.SchemaName = resolveSchema(b, offline.CatalogName, offline.SchemaName) + offline.CatalogName = resolveCatalog(b, offline.CatalogName) + } + if online := member.OnlineStoreConfig; online != nil { + online.SchemaName = resolveSchema(b, online.CatalogName, online.SchemaName) + online.CatalogName = resolveCatalog(b, online.CatalogName) + } + } + } for _, mse := range b.Config.Resources.ModelServingEndpoints { if mse == nil { continue diff --git a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go index 347f0b72caa..15094079ee2 100644 --- a/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go +++ b/bundle/config/mutator/resourcemutator/capture_uc_dependencies_test.go @@ -365,6 +365,7 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { QualityMonitors: map[string]*resources.QualityMonitor{"nil": nil, "empty": {}}, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{"nil": nil, "empty": {}}, VectorSearchIndexes: map[string]*resources.VectorSearchIndex{"nil": nil, "empty": {}}, + MaterializationSets: map[string]*resources.MaterializationSet{"nil": nil, "empty": {}}, }, }, } @@ -373,6 +374,72 @@ func TestCaptureUCDependenciesNilResources(t *testing.T) { require.Nil(t, d) } +func TestCaptureUCDependenciesMaterializationSet(t *testing.T) { + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + Catalogs: map[string]*resources.Catalog{ + "my_catalog": {CreateCatalog: catalog.CreateCatalog{Name: "mycatalog"}}, + }, + Schemas: map[string]*resources.Schema{ + "my_schema": {CreateSchema: catalog.CreateSchema{CatalogName: "mycatalog", Name: "myschema"}}, + }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "set1": {MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + { + FeatureName: "mycatalog.myschema.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "mycatalog", SchemaName: "myschema", TableNamePrefix: "feat_off", + }, + }, + { + FeatureName: "mycatalog.myschema.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{ + OnlineStoreName: "store", CatalogName: "mycatalog", + SchemaName: "myschema", TableNamePrefix: "feat", + }, + }, + // Neither destination declared in the bundle, so both pass through. + { + FeatureName: "other.other.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "other", SchemaName: "other", TableNamePrefix: "feat_off", + }, + }, + // A member with no destination at all. + {FeatureName: "mycatalog.myschema.feat"}, + }, + }}, + "nil_set": nil, + }, + }, + }, + } + + d := bundle.Apply(t.Context(), b, CaptureUCDependencies()) + require.Nil(t, d) + + schemaRef := "${resources.schemas.my_schema.name}" + catalogRef := "${resources.catalogs.my_catalog.name}" + members := b.Config.Resources.MaterializationSets["set1"].MaterializedFeatures + + assert.Equal(t, catalogRef, members[0].OfflineStoreConfig.CatalogName) + assert.Equal(t, schemaRef, members[0].OfflineStoreConfig.SchemaName) + assert.Equal(t, "feat_off", members[0].OfflineStoreConfig.TableNamePrefix) + + assert.Equal(t, catalogRef, members[1].OnlineStoreConfig.CatalogName) + assert.Equal(t, schemaRef, members[1].OnlineStoreConfig.SchemaName) + assert.Equal(t, "store", members[1].OnlineStoreConfig.OnlineStoreName) + + assert.Equal(t, "other", members[2].OfflineStoreConfig.CatalogName) + assert.Equal(t, "other", members[2].OfflineStoreConfig.SchemaName) + + assert.Nil(t, members[3].OfflineStoreConfig) + assert.Nil(t, members[3].OnlineStoreConfig) + assert.Nil(t, b.Config.Resources.MaterializationSets["nil_set"]) +} + func TestSplitUCName(t *testing.T) { tests := []struct { name string diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go new file mode 100644 index 00000000000..82a94d54b41 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members.go @@ -0,0 +1,125 @@ +package resourcemutator + +import ( + "context" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/diag" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +type expandMaterializationSetMaterializedFeatureMembers struct{} + +func ExpandMaterializationSetMaterializedFeatureMembers() bundle.Mutator { + return &expandMaterializationSetMaterializedFeatureMembers{} +} + +func (m *expandMaterializationSetMaterializedFeatureMembers) Name() string { + return "ExpandMaterializationSetMaterializedFeatureMembers" +} + +// sawtoothMaterializedFeatureMembers builds the three materialized features a sawtooth-window +// feature needs to declare for its materialization: a batch offline materialized feature, a batch +// online materialized feature, and a streaming online materialized feature, where the batch offline +// and batch online use a DERIVED cron schedule and the streaming online and batch online must have +// the same online destination. +// +// materialized_feature_id is absent here because it is assigned server side. +func sawtoothMaterializedFeatureMembers(entry resources.SawtoothMaterializedFeatures) []ml.MaterializedFeature { + return []ml.MaterializedFeature{ + // Batch offline. + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + }, + // Batch online. + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OnlineStoreConfig: entry.OnlineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + }, + // Streaming online. + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OnlineStoreConfig: entry.OnlineStoreConfig, + StreamingMode: entry.StreamingMode, + }, + } +} + +// batchOfflineAndOnlineMaterializedFeatureMembers builds two materializations of the specified +// feature to the online and offline destination. +func batchOfflineAndOnlineMaterializedFeatureMembers(entry resources.BatchOfflineAndOnlineMaterializedFeatures) []ml.MaterializedFeature { + return []ml.MaterializedFeature{ + // Batch offline. + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + }, + // Batch online. + { + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OnlineStoreConfig: entry.OnlineStoreConfig, + CronSchedule: entry.CronSchedule, + CronScheduleTrigger: entry.CronScheduleTrigger, + TableTrigger: entry.TableTrigger, + }, + } +} + +// Apply mutates typed state, and the Python path merges the dynamic tree back once the normalize +// sequence returns, so this must not be the last entry in applyNormalizeMutators: that tree picks up +// a typed mutation only when the next mutator enters. +func (m *expandMaterializationSetMaterializedFeatureMembers) Apply(_ context.Context, b *bundle.Bundle) diag.Diagnostics { + for _, set := range b.Config.Resources.MaterializationSets { + if set == nil { + continue + } + + // Append rather than assign because a set can declare one feature in shorthand and another + // as individual materialized feature members. The adapter keys materialized feature members + // by content rather than by position. + for _, entry := range set.SawtoothMaterializedFeatures { + set.MaterializedFeatures = append(set.MaterializedFeatures, + sawtoothMaterializedFeatureMembers(entry)...) + } + for _, entry := range set.BatchOfflineAndOnlineMaterializedFeatures { + set.MaterializedFeatures = append(set.MaterializedFeatures, + batchOfflineAndOnlineMaterializedFeatureMembers(entry)...) + } + + set.SawtoothMaterializedFeatures = nil + set.BatchOfflineAndOnlineMaterializedFeatures = nil + } + + return nil +} diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go new file mode 100644 index 00000000000..17adc6eadd7 --- /dev/null +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_coverage_test.go @@ -0,0 +1,137 @@ +package resourcemutator_test + +import ( + "maps" + "reflect" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// serverOwnedMemberFields are the member fields no authoring type declares. All but the first are +// OUTPUT_ONLY in the spec, so the schema generator keeps them out of the bundle schema; +// materialized_feature_id is left out because the server assigns one per member and a single +// declared id cannot belong to all of them. +var serverOwnedMemberFields = map[string]bool{ + "MaterializedFeatureId": true, + "MaterializationSetId": true, + "IsOnline": true, + "JobId": true, + "PipelineId": true, + "TableName": true, + "LastMaterializationTime": true, + "LatestBackfillOperation": true, +} + +// memberFieldsDerivedByConfigLayer are neither declared nor carried: the dyn-to-typed conversion +// rebuilds them per member from the fields that member holds. ForceSendFields comes back naming +// each expanded member's own pointer fields, so an explicitly set zero value survives the +// expansion without the mutator naming it. +var memberFieldsDerivedByConfigLayer = map[string]bool{ + "ForceSendFields": true, +} + +// entryWithEveryFieldSet returns an authoring entry whose every field holds a non-zero value, so +// that a field reaching no expanded member is visible as a zero. +func entryWithEveryFieldSet[T any](t *testing.T) T { + t.Helper() + + var entry T + value := reflect.ValueOf(&entry).Elem() + for field := range reflect.TypeFor[T]().Fields() { + target := value.FieldByName(field.Name) + switch field.Type.Kind() { + case reflect.String: + target.SetString("x") + case reflect.Bool: + target.SetBool(true) + case reflect.Int, reflect.Int32, reflect.Int64: + target.SetInt(1) + case reflect.Pointer: + target.Set(reflect.New(field.Type.Elem())) + case reflect.Map: + pair := reflect.MakeMap(field.Type) + pair.SetMapIndex(reflect.ValueOf("k"), reflect.ValueOf("v")) + target.Set(pair) + case reflect.Slice: + target.Set(reflect.MakeSlice(field.Type, 1, 1)) + default: + require.Failf(t, "unhandled field kind", + "field %s has kind %s, which this test does not know how to populate", + field.Name, field.Type.Kind()) + } + } + return entry +} + +func notCarried(extra ...string) map[string]bool { + out := maps.Clone(serverOwnedMemberFields) + for _, name := range extra { + out[name] = true + } + return out +} + +// TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField keeps each +// expansion's field list exhaustive and each authoring type complete against the API. Members are +// built by naming their fields, and the authoring types are hand-written, so a member field added +// to the proto later reaches no expanded member and nothing else notices. +func TestExpandMaterializationSetMaterializedFeatureMembersCoversEveryMemberField(t *testing.T) { + for name, tc := range map[string]struct { + set *resources.MaterializationSet + notCarried map[string]bool + }{ + "sawtooth_materialized_features": { + set: &resources.MaterializationSet{ + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{ + entryWithEveryFieldSet[resources.SawtoothMaterializedFeatures](t), + }, + }, + notCarried: notCarried(), + }, + "batch_offline_and_online_materialized_features": { + set: &resources.MaterializationSet{ + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{ + entryWithEveryFieldSet[resources.BatchOfflineAndOnlineMaterializedFeatures](t), + }, + }, + // A feature on the batch path has no streaming mode, so the type declares none. + notCarried: notCarried("StreamingMode"), + }, + } { + t.Run(name, func(t *testing.T) { + expanded := expandSet(t, tc.set) + require.NotEmpty(t, expanded.MaterializedFeatures) + + for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + if memberFieldsDerivedByConfigLayer[field.Name] { + continue + } + + carried := false + for _, member := range expanded.MaterializedFeatures { + if !reflect.ValueOf(member).FieldByName(field.Name).IsZero() { + carried = true + break + } + } + + if tc.notCarried[field.Name] { + assert.Falsef(t, carried, + "field %s is listed as not carried but reached an expanded member; drop it "+ + "from that list or stop carrying it", field.Name) + continue + } + assert.Truef(t, carried, + "field %s reached no member of the %s expansion. Declare it on the authoring "+ + "type in bundle/config/resources/materialization_set.go and carry it in "+ + "expand_materialization_set_materialized_feature_members.go, or list it as "+ + "not carried with the reason it is server-owned", + field.Name, name) + } + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go new file mode 100644 index 00000000000..6e2666b153a --- /dev/null +++ b/bundle/config/mutator/resourcemutator/expand_materialization_set_materialized_feature_members_test.go @@ -0,0 +1,226 @@ +package resourcemutator_test + +import ( + "testing" + + "github.com/databricks/cli/bundle" + "github.com/databricks/cli/bundle/config" + "github.com/databricks/cli/bundle/config/mutator/resourcemutator" + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + shorthandFeatureName = "main.myschema.feat" + shorthandCronExpr = "0 0 * * * ?" +) + +func shorthandOfflineStore() *ml.OfflineStoreConfig { + return &ml.OfflineStoreConfig{ + CatalogName: "main", + SchemaName: "myschema", + TableNamePrefix: "feat_off", + } +} + +func shorthandOnlineStore() *ml.OnlineStoreConfig { + return &ml.OnlineStoreConfig{ + OnlineStoreName: "store", + CatalogName: "main", + SchemaName: "myschema", + TableNamePrefix: "feat", + } +} + +func expandSet(t *testing.T, set *resources.MaterializationSet) *resources.MaterializationSet { + t.Helper() + + b := &bundle.Bundle{ + Config: config.Root{ + Resources: config.Resources{ + MaterializationSets: map[string]*resources.MaterializationSet{"set1": set}, + }, + }, + } + diags := bundle.Apply(t.Context(), b, resourcemutator.ExpandMaterializationSetMaterializedFeatureMembers()) + require.NoError(t, diags.Error()) + + return b.Config.Resources.MaterializationSets["set1"] +} + +func TestExpandMaterializationSetMaterializedFeatureMembersSawtooth(t *testing.T) { + set := expandSet(t, &resources.MaterializationSet{ + MaterializationSet: ml.MaterializationSet{}, + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 3) + + offlineBatch := set.MaterializedFeatures[0] + assert.Equal(t, shorthandOfflineStore(), offlineBatch.OfflineStoreConfig) + assert.Nil(t, offlineBatch.OnlineStoreConfig) + assert.Equal(t, ml.CronScheduleModeDerived, offlineBatch.CronScheduleTrigger.Mode) + assert.Nil(t, offlineBatch.StreamingMode) + + onlineBatch := set.MaterializedFeatures[1] + assert.Nil(t, onlineBatch.OfflineStoreConfig) + assert.Equal(t, shorthandOnlineStore(), onlineBatch.OnlineStoreConfig) + assert.Equal(t, ml.CronScheduleModeDerived, onlineBatch.CronScheduleTrigger.Mode) + assert.Nil(t, onlineBatch.StreamingMode) + + onlineStreaming := set.MaterializedFeatures[2] + assert.Nil(t, onlineStreaming.OfflineStoreConfig) + assert.Equal(t, shorthandOnlineStore(), onlineStreaming.OnlineStoreConfig) + assert.Nil(t, onlineStreaming.CronScheduleTrigger) + assert.Equal(t, ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm, onlineStreaming.StreamingMode.Mode) + + for _, member := range set.MaterializedFeatures { + assert.Equal(t, shorthandFeatureName, member.FeatureName) + } + assert.Nil(t, set.SawtoothMaterializedFeatures) +} + +func TestExpandMaterializationSetMaterializedFeatureMembersBatchOfflineAndOnline(t *testing.T) { + trigger := &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr} + set := expandSet(t, &resources.MaterializationSet{ + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: trigger, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 2) + + offline := set.MaterializedFeatures[0] + assert.Equal(t, shorthandOfflineStore(), offline.OfflineStoreConfig) + assert.Nil(t, offline.OnlineStoreConfig) + + online := set.MaterializedFeatures[1] + assert.Nil(t, online.OfflineStoreConfig) + assert.Equal(t, shorthandOnlineStore(), online.OnlineStoreConfig) + + // The trigger is not split for a batch pair: both members refresh together. + assert.Equal(t, trigger, offline.CronScheduleTrigger) + assert.Equal(t, trigger, online.CronScheduleTrigger) + assert.Nil(t, set.BatchOfflineAndOnlineMaterializedFeatures) +} + +func TestExpandMaterializationSetMaterializedFeatureMembersCarriesSharedFields(t *testing.T) { + notifications := &ml.MaterializationNotifications{ + OnFailure: &ml.MaterializationFailureNotification{EmailAddresses: []string{"a@example.com"}}, + } + entry := ml.MaterializedFeature{ + FeatureName: shorthandFeatureName, + Tags: map[string]string{"team": "feature-store"}, + BudgetPolicyId: "budget-1", + Notifications: notifications, + PipelineScheduleState: ml.MaterializedFeaturePipelineScheduleStatePaused, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + } + + for name, set := range map[string]*resources.MaterializationSet{ + "sawtooth": {SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{{ + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + OnlineStoreConfig: entry.OnlineStoreConfig, + }}}, + "batch pair": {BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{{ + FeatureName: entry.FeatureName, + Tags: entry.Tags, + BudgetPolicyId: entry.BudgetPolicyId, + Notifications: entry.Notifications, + PipelineScheduleState: entry.PipelineScheduleState, + OfflineStoreConfig: entry.OfflineStoreConfig, + OnlineStoreConfig: entry.OnlineStoreConfig, + }}}, + } { + t.Run(name, func(t *testing.T) { + expanded := expandSet(t, set) + require.NotEmpty(t, expanded.MaterializedFeatures) + + for _, member := range expanded.MaterializedFeatures { + assert.Equal(t, shorthandFeatureName, member.FeatureName) + assert.Equal(t, map[string]string{"team": "feature-store"}, member.Tags) + assert.Equal(t, "budget-1", member.BudgetPolicyId) + assert.Equal(t, notifications, member.Notifications) + assert.Equal(t, ml.MaterializedFeaturePipelineScheduleStatePaused, member.PipelineScheduleState) + } + }) + } +} + +func TestExpandMaterializationSetMaterializedFeatureMembersAppendsToDeclaredMembers(t *testing.T) { + declared := ml.MaterializedFeature{ + FeatureName: "main.myschema.other", + OfflineStoreConfig: shorthandOfflineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr}, + } + set := expandSet(t, &resources.MaterializationSet{ + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{declared}, + }, + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeDerived}, + StreamingMode: &ml.StreamingMode{Mode: ml.StreamingModeStreamingModeTypeStreamingModeTypeRtm}, + }}, + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{{ + FeatureName: "main.myschema.pair", + OfflineStoreConfig: shorthandOfflineStore(), + OnlineStoreConfig: shorthandOnlineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr}, + }}, + }) + + require.Len(t, set.MaterializedFeatures, 6) + assert.Equal(t, declared, set.MaterializedFeatures[0]) + assert.Nil(t, set.SawtoothMaterializedFeatures) + assert.Nil(t, set.BatchOfflineAndOnlineMaterializedFeatures) +} + +func TestExpandMaterializationSetMaterializedFeatureMembersLeavesASetWithoutShorthandAlone(t *testing.T) { + declared := ml.MaterializedFeature{ + FeatureName: shorthandFeatureName, + OfflineStoreConfig: shorthandOfflineStore(), + CronScheduleTrigger: &ml.CronSchedule{Mode: ml.CronScheduleModeManual, CronExpression: shorthandCronExpr}, + } + + for name, set := range map[string]*resources.MaterializationSet{ + "no shorthand field": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{declared}, + }, + }, + "empty shorthand field": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{declared}, + }, + SawtoothMaterializedFeatures: []resources.SawtoothMaterializedFeatures{}, + BatchOfflineAndOnlineMaterializedFeatures: []resources.BatchOfflineAndOnlineMaterializedFeatures{}, + }, + } { + t.Run(name, func(t *testing.T) { + expanded := expandSet(t, set) + assert.Equal(t, []ml.MaterializedFeature{declared}, expanded.MaterializedFeatures) + }) + } +} diff --git a/bundle/config/mutator/resourcemutator/resource_mutator.go b/bundle/config/mutator/resourcemutator/resource_mutator.go index 2262ad6c0a5..c14aa127576 100644 --- a/bundle/config/mutator/resourcemutator/resource_mutator.go +++ b/bundle/config/mutator/resourcemutator/resource_mutator.go @@ -149,6 +149,12 @@ func applyNormalizeMutators(ctx context.Context, b *bundle.Bundle) { // Expands glob patterns in pipeline library paths to include all matching files ExpandPipelineGlobPaths(), + // Reads (typed): resources.materialization_sets.*.{sawtooth,batch_offline_and_online}_materialized_features + // Updates (typed): resources.materialization_sets.*.materialized_features (appends the members each entry is made of) + // Clears (typed): both shorthand fields, after the expansion + // Expands each shorthand entry into the several materialized features one feature's materialization is made of + ExpandMaterializationSetMaterializedFeatureMembers(), + // Reads (dynamic): resources.jobs.*.job_clusters (reads job clusters to merge) // Updates (dynamic): resources.jobs.*.job_clusters (merges job clusters with the same job_cluster_key) MergeJobClusters(), diff --git a/bundle/config/mutator/resourcemutator/run_as_test.go b/bundle/config/mutator/resourcemutator/run_as_test.go index 200939f2dd0..15b303bd7c7 100644 --- a/bundle/config/mutator/resourcemutator/run_as_test.go +++ b/bundle/config/mutator/resourcemutator/run_as_test.go @@ -54,6 +54,7 @@ func allResourceTypes(t *testing.T) []string { "internal_immutable_snapshots", "job_runs", "jobs", + "materialization_sets", "mcp_services", "model_provider_services", "model_services", @@ -188,6 +189,7 @@ var allowList = []string{ "external_locations", "synced_database_tables", "jobs", + "materialization_sets", "pipelines", "models", "model_services", diff --git a/bundle/config/resources.go b/bundle/config/resources.go index 75fef099133..77b56bb6144 100644 --- a/bundle/config/resources.go +++ b/bundle/config/resources.go @@ -17,6 +17,7 @@ type Resources struct { Pipelines map[string]*resources.Pipeline `json:"pipelines,omitempty"` Features map[string]*resources.Feature `json:"features,omitempty"` + MaterializationSets map[string]*resources.MaterializationSet `json:"materialization_sets,omitempty"` Models map[string]*resources.MlflowModel `json:"models,omitempty"` Experiments map[string]*resources.MlflowExperiment `json:"experiments,omitempty"` ModelServingEndpoints map[string]*resources.ModelServingEndpoint `json:"model_serving_endpoints,omitempty"` @@ -116,6 +117,7 @@ func (r *Resources) AllResources() []ResourceGroup { collectResourceMap(descriptions["models"], r.Models), collectResourceMap(descriptions["experiments"], r.Experiments), collectResourceMap(descriptions["features"], r.Features), + collectResourceMap(descriptions["materialization_sets"], r.MaterializationSets), collectResourceMap(descriptions["model_serving_endpoints"], r.ModelServingEndpoints), collectResourceMap(descriptions["model_services"], r.ModelServices), collectResourceMap(descriptions["mcp_services"], r.McpServices), @@ -203,6 +205,7 @@ func SupportedResources() map[string]resources.ResourceDescription { "models": (&resources.MlflowModel{}).ResourceDescription(), "experiments": (&resources.MlflowExperiment{}).ResourceDescription(), "features": (&resources.Feature{}).ResourceDescription(), + "materialization_sets": (&resources.MaterializationSet{}).ResourceDescription(), "instance_pools": (&resources.InstancePool{}).ResourceDescription(), "model_serving_endpoints": (&resources.ModelServingEndpoint{}).ResourceDescription(), "model_services": (&resources.ModelService{}).ResourceDescription(), diff --git a/bundle/config/resources/feature.go b/bundle/config/resources/feature.go index 05d27624a2e..715bcddbd9b 100644 --- a/bundle/config/resources/feature.go +++ b/bundle/config/resources/feature.go @@ -17,6 +17,144 @@ type Feature struct { BaseResource ID string `json:"id,omitempty" bundle:"readonly"` ml.Feature + + // ========================================= + // === overrides over [ml.Feature] ========= + // ========================================= + + Function Function `json:"function"` + Source DataSource `json:"source"` + TimeWindow *TimeWindow `json:"time_window,omitempty"` +} + +// The types below re-type the API's duration fields as strings so a bundle can write a duration +// with day and hour units; featureState converts them before deploy. Each embeds the API type and +// shadows only the fields it re-types, so every other field resolves through the embedding. + +type Function struct { + ml.Function + AggregationFunction *AggregationFunction `json:"aggregation_function,omitempty"` +} + +type AggregationFunction struct { + ml.AggregationFunction + TimeWindow *TimeWindow `json:"time_window,omitempty"` +} + +type TimeWindow struct { + ml.TimeWindow + Rolling *RollingWindow `json:"rolling,omitempty"` + Sawtooth *SawtoothWindow `json:"sawtooth,omitempty"` + Sliding *SlidingWindow `json:"sliding,omitempty"` + Tumbling *TumblingWindow `json:"tumbling,omitempty"` +} + +type RollingWindow struct { + ml.RollingWindow + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type SawtoothWindow struct { + ml.SawtoothWindow + Delay string `json:"delay,omitempty"` + WindowDuration string `json:"window_duration,omitempty"` +} + +type SlidingWindow struct { + ml.SlidingWindow + Delay string `json:"delay,omitempty"` + Offset string `json:"offset,omitempty"` +} + +type TumblingWindow struct { + ml.TumblingWindow + Delay string `json:"delay,omitempty"` + Offset string `json:"offset,omitempty"` +} + +type DataSource struct { + ml.DataSource + Lateness *SourceLateness `json:"lateness,omitempty"` +} + +type SourceLateness struct { + ml.SourceLateness + SettlingDelay string `json:"settling_delay,omitempty"` +} + +// Each wrapper declares its own JSON methods, overriding the ones its embedded API type promotes. + +func (c *Function) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c Function) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *AggregationFunction) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c AggregationFunction) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *TimeWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c TimeWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *RollingWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c RollingWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *SawtoothWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c SawtoothWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *SlidingWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c SlidingWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *TumblingWindow) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c TumblingWindow) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *DataSource) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c DataSource) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) +} + +func (c *SourceLateness) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, c) +} + +func (c SourceLateness) MarshalJSON() ([]byte, error) { + return marshal.Marshal(c) } func (f *Feature) UnmarshalJSON(b []byte) error { diff --git a/bundle/config/resources/materialization_set.go b/bundle/config/resources/materialization_set.go new file mode 100644 index 00000000000..3d72cf45f18 --- /dev/null +++ b/bundle/config/resources/materialization_set.go @@ -0,0 +1,129 @@ +package resources + +import ( + "context" + "net/url" + + "github.com/databricks/cli/libs/log" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/marshal" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// MaterializationSet is the bundle config for a group of materialized features created and +// managed as one unit. +type MaterializationSet struct { + BaseResource + ID string `json:"id,omitempty" bundle:"readonly"` + ml.MaterializationSet + + // SawtoothMaterializedFeatures declares, per entry, the whole materialization of one + // sawtooth-window feature: an offline batch member, an online batch member and an online + // streaming member. ExpandMaterializationSetMaterializedFeatureMembers fans each entry out + // into three materialized features. + SawtoothMaterializedFeatures []SawtoothMaterializedFeatures `json:"sawtooth_materialized_features,omitempty"` + + // BatchOfflineAndOnlineMaterializedFeatures declares, per entry, one feature materialized to + // the online and offline destination. ExpandMaterializationSetMaterializedFeatureMembers fans + // each entry out into two MaterializedFeatures. + BatchOfflineAndOnlineMaterializedFeatures []BatchOfflineAndOnlineMaterializedFeatures `json:"batch_offline_and_online_materialized_features,omitempty"` +} + +// SawtoothMaterializedFeatures is the materialized features of one sawtooth-window feature, which +// the author declares once. ml.MaterializedFeature cannot stand in for it: a materialized feature +// holds one destination and one trigger, both declared as proto oneofs, while this holds both +// destinations and both trigger kinds and the expansion splits them across the three members. +// +// materialized_feature_id is absent because the server assigns one per member. +type SawtoothMaterializedFeatures struct { + FeatureName string `json:"feature_name"` + Tags map[string]string `json:"tags,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` + Notifications *ml.MaterializationNotifications `json:"notifications,omitempty"` + PipelineScheduleState ml.MaterializedFeaturePipelineScheduleState `json:"pipeline_schedule_state,omitempty"` + + OfflineStoreConfig *ml.OfflineStoreConfig `json:"offline_store_config,omitempty"` + OnlineStoreConfig *ml.OnlineStoreConfig `json:"online_store_config,omitempty"` + + // The batch members take the cron schedule and the table trigger; the streaming member takes + // the streaming mode. + CronSchedule string `json:"cron_schedule,omitempty"` + CronScheduleTrigger *ml.CronSchedule `json:"cron_schedule_trigger,omitempty"` + TableTrigger *ml.TableTrigger `json:"table_trigger,omitempty"` + StreamingMode *ml.StreamingMode `json:"streaming_mode,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +// BatchOfflineAndOnlineMaterializedFeatures is the materialized features of one feature taken to +// both stores on the batch path, which the author declares once. It carries no streaming mode: a +// feature on the batch path has none, and the two members it expands into differ only in their +// destination. +// +// materialized_feature_id is absent because the server assigns one per member. +type BatchOfflineAndOnlineMaterializedFeatures struct { + FeatureName string `json:"feature_name"` + Tags map[string]string `json:"tags,omitempty"` + BudgetPolicyId string `json:"budget_policy_id,omitempty"` + Notifications *ml.MaterializationNotifications `json:"notifications,omitempty"` + PipelineScheduleState ml.MaterializedFeaturePipelineScheduleState `json:"pipeline_schedule_state,omitempty"` + + OfflineStoreConfig *ml.OfflineStoreConfig `json:"offline_store_config,omitempty"` + OnlineStoreConfig *ml.OnlineStoreConfig `json:"online_store_config,omitempty"` + + CronSchedule string `json:"cron_schedule,omitempty"` + CronScheduleTrigger *ml.CronSchedule `json:"cron_schedule_trigger,omitempty"` + TableTrigger *ml.TableTrigger `json:"table_trigger,omitempty"` + + ForceSendFields []string `json:"-" url:"-"` +} + +func (s *MaterializationSet) UnmarshalJSON(b []byte) error { + return marshal.Unmarshal(b, s) +} + +func (s MaterializationSet) MarshalJSON() ([]byte, error) { + return marshal.Marshal(s) +} + +func (s *MaterializationSet) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) { + _, err := w.FeatureEngineering.GetMaterializationSet(ctx, ml.GetMaterializationSetRequest{ + MaterializationSetId: id, + }) + if err != nil { + log.Debugf(ctx, "materialization set %s does not exist: %v", id, err) + + if apierr.IsMissing(err) { + return false, nil + } + + return false, err + } + + return true, nil +} + +func (*MaterializationSet) ResourceDescription() ResourceDescription { + return ResourceDescription{ + SingularName: "materialization_set", + PluralName: "materialization_sets", + SingularTitle: "Materialization Set", + PluralTitle: "Materialization Sets", + } +} + +// GetName returns the in-product name, which is empty: a set is identified only by its +// server-assigned id. +func (s *MaterializationSet) GetName() string { + return "" +} + +// GetURL reports that this type has no workspace URL: a set has no standalone page, and its +// members surface under their own features. +func (s *MaterializationSet) GetURL() (string, bool) { + return "", false +} + +// InitializeURL leaves URL unset. See GetURL. +func (s *MaterializationSet) InitializeURL(_ url.URL) {} diff --git a/bundle/config/resources/materialization_set_test.go b/bundle/config/resources/materialization_set_test.go new file mode 100644 index 00000000000..643ceac4a7c --- /dev/null +++ b/bundle/config/resources/materialization_set_test.go @@ -0,0 +1,25 @@ +package resources + +import ( + "net/url" + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestMaterializationSetReportsNoWorkspaceURL pins the signal that keeps `bundle summary` from +// printing a URL line for a set. BaseResource.GetURL returns ("", true), which the summary reads as +// a supported type awaiting deployment and renders as "(not deployed)". +func TestMaterializationSetReportsNoWorkspaceURL(t *testing.T) { + set := &MaterializationSet{ID: "0123456789abcdef0123456789abcdef"} + set.URL = "https://example.cloud.databricks.com/explore/data/features" + + rawURL, supportsURL := set.GetURL() + assert.False(t, supportsURL) + assert.Empty(t, rawURL) + + set.InitializeURL(url.URL{Scheme: "https", Host: "example.cloud.databricks.com"}) + rawURL, supportsURL = set.GetURL() + assert.False(t, supportsURL, "InitializeURL must not make the type look URL-bearing") + assert.Empty(t, rawURL) +} diff --git a/bundle/config/resources_test.go b/bundle/config/resources_test.go index 963e7efd4ff..4cdafc52747 100644 --- a/bundle/config/resources_test.go +++ b/bundle/config/resources_test.go @@ -128,6 +128,9 @@ func TestBundleResourcePluralNamesResolveInWorkspaceURLs(t *testing.T) { // Resources that intentionally have no workspace URL. noURL := map[string]bool{ "external_locations": true, + // A materialization set has no standalone workspace page; its members + // surface under their own features. + "materialization_sets": true, // A job run does have a workspace URL, but it's addressed by two IDs // (job + run) so it can't be expressed as a single-ID pattern here; it's // built in JobRun.InitializeURL via workspaceurls.JobRunURL instead. @@ -368,6 +371,15 @@ func TestResourcesBindSupport(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "my_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.myschema.my_feature"}, + }, + }, + }, + }, VectorSearchEndpoints: map[string]*resources.VectorSearchEndpoint{ "my_vector_search_endpoint": { CreateEndpoint: vectorsearch.CreateEndpoint{ @@ -399,6 +411,7 @@ func TestResourcesBindSupport(t *testing.T) { m.GetMockPipelinesAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExperimentsAPI().EXPECT().GetExperiment(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockFeatureEngineeringAPI().EXPECT().GetFeature(mock.Anything, mock.Anything).Return(nil, nil) + m.GetMockFeatureEngineeringAPI().EXPECT().GetMaterializationSet(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockRegisteredModelsAPI().EXPECT().Get(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockCatalogsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) m.GetMockExternalLocationsAPI().EXPECT().GetByName(mock.Anything, mock.Anything).Return(nil, nil) diff --git a/bundle/direct/dresources/all.go b/bundle/direct/dresources/all.go index 88f08048235..b13465550ca 100644 --- a/bundle/direct/dresources/all.go +++ b/bundle/direct/dresources/all.go @@ -12,6 +12,7 @@ var SupportedResources = map[string]any{ "pipelines": (*ResourcePipeline)(nil), "experiments": (*ResourceExperiment)(nil), "features": (*ResourceFeature)(nil), + "materialization_sets": (*ResourceMaterializationSet)(nil), "catalogs": (*ResourceCatalog)(nil), "schemas": (*ResourceSchema)(nil), "external_locations": (*ResourceExternalLocation)(nil), diff --git a/bundle/direct/dresources/all_test.go b/bundle/direct/dresources/all_test.go index 1b0da0602a2..2b595f331ac 100644 --- a/bundle/direct/dresources/all_test.go +++ b/bundle/direct/dresources/all_test.go @@ -123,6 +123,31 @@ var testConfig map[string]any = map[string]any{ TimeseriesColumn: &ml.TimeseriesColumn{Name: "ts"}, }, + // A feature materialized to both an offline table and one online store: the two members + // share a feature name, so they exercise the member key's online-store discriminator. + "materialization_sets": &resources.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + { + FeatureName: "main.default.my_feature", + CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, + OfflineStoreConfig: &ml.OfflineStoreConfig{ + CatalogName: "main", + SchemaName: "default", + TableNamePrefix: "offline", + }, + }, + { + FeatureName: "main.default.my_feature", + CronScheduleTrigger: &ml.CronSchedule{CronExpression: "0 0 * * * ?"}, + OnlineStoreConfig: &ml.OnlineStoreConfig{ + CatalogName: "main", + SchemaName: "default", + OnlineStoreName: "my_store", + }, + }, + }, + }, + "experiments": &resources.MlflowExperiment{ Name: "my-experiment", Tags: []ml.ExperimentTag{ @@ -943,22 +968,27 @@ func TestAll(t *testing.T) { // testIgnoreFilter encapsulates the logic for filtering fields based on ignore_remote_changes config. type testIgnoreFilter struct { - ignoreFields map[string]bool - adapter *Adapter + ignoreFields map[string]bool + ignorePatterns []*structpath.PatternNode + adapter *Adapter } // newTestIgnoreFilter creates a filter from the adapter's resource configs. // It also ignores fields that exist in StateType but not in RemoteType, because // those are automatically suppressed by the planner (reason: missing_in_remote) // and RemapState cannot populate them from remote state. -func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { +func newTestIgnoreFilter(adapter *Adapter, group string) *testIgnoreFilter { ignoreFields := make(map[string]bool) + var ignorePatterns []*structpath.PatternNode for _, cfg := range []*ResourceLifecycleConfig{adapter.ResourceConfig(), adapter.GeneratedResourceConfig()} { if cfg == nil { continue } for _, p := range cfg.IgnoreRemoteChanges { ignoreFields[p.Field.String()] = true + if group == "materialization_sets" { + ignorePatterns = append(ignorePatterns, p.Field) + } } } // Auto-include fields present in StateType but absent from RemoteType. @@ -972,7 +1002,7 @@ func newTestIgnoreFilter(adapter *Adapter) *testIgnoreFilter { } return true }) - return &testIgnoreFilter{ignoreFields: ignoreFields, adapter: adapter} + return &testIgnoreFilter{ignoreFields: ignoreFields, ignorePatterns: ignorePatterns, adapter: adapter} } // shouldIgnore returns true if the field at the given path should be ignored. @@ -991,7 +1021,24 @@ func (f *testIgnoreFilter) shouldIgnore(path string) bool { topLevelField = parts[0] } - return f.ignoreFields[topLevelField] + if f.ignoreFields[topLevelField] { + return true + } + + // Pattern rules are collected for materialization_sets alone, so this loop is empty for every + // other resource. A rule inside a slice (foo[*].bar) never matches the indexed path a walk + // produces (foo[0].bar); jobs, apps and database_instances carry such rules too, so widening + // the comparison to them is its own change. + parsed, err := structpath.ParsePath(path) + if err != nil { + return false + } + for _, pattern := range f.ignorePatterns { + if parsed.HasPatternPrefix(pattern) { + return true + } + } + return false } // filterChanges returns only the changes that should not be ignored. @@ -1082,7 +1129,7 @@ func testCRUD(t *testing.T, group string, adapter *Adapter, client *databricks.W require.NotNil(t, remappedState) // Create filter for fields that should be ignored based on ignore_remote_changes config. - ignoreFilter := newTestIgnoreFilter(adapter) + ignoreFilter := newTestIgnoreFilter(adapter, group) if remoteStateFromCreate != nil { remappedRemoteStateFromCreate, err := adapter.RemapState(remoteStateFromCreate) diff --git a/bundle/direct/dresources/configs/materialization_sets.yml b/bundle/direct/dresources/configs/materialization_sets.yml new file mode 100644 index 00000000000..992130fec37 --- /dev/null +++ b/bundle/direct/dresources/configs/materialization_sets.yml @@ -0,0 +1,24 @@ +recreate_on_changes: + - field: materialized_features[*].feature_name + reason: immutable + - field: materialized_features[*].cron_schedule + reason: immutable + - field: materialized_features[*].cron_schedule_trigger + reason: immutable + - field: materialized_features[*].table_trigger + reason: immutable + - field: materialized_features[*].streaming_mode + reason: immutable + - field: materialized_features[*].pipeline_schedule_state + reason: immutable + - field: materialized_features[*].tags + reason: immutable + - field: materialized_features[*].budget_policy_id + reason: immutable + - field: materialized_features[*].notifications + reason: immutable + +backend_defaults: + # A member the config leaves unset still reports the state the server set on create, which the + # bundle never wrote, so the undeclared case diffs as old=nil, new=nil, remote=ACTIVE. + - field: materialized_features[*].pipeline_schedule_state diff --git a/bundle/direct/dresources/feature.go b/bundle/direct/dresources/feature.go index 8d95e3d0eac..b10f78d2c0d 100644 --- a/bundle/direct/dresources/feature.go +++ b/bundle/direct/dresources/feature.go @@ -5,6 +5,7 @@ import ( "slices" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structvar" "github.com/databricks/databricks-sdk-go" "github.com/databricks/databricks-sdk-go/service/ml" ) @@ -20,7 +21,18 @@ func (*ResourceFeature) New(client *databricks.WorkspaceClient) *ResourceFeature } func (*ResourceFeature) PrepareState(input *resources.Feature) *ml.Feature { - return &input.Feature + // An unparseable duration is reported by PrepareInputConfig, which runs first. + state, _ := featureState(input) + return state +} + +// PrepareInputConfig reports a duration featureState cannot parse, which PrepareState cannot: its +// signature carries no error. +func (*ResourceFeature) PrepareInputConfig(input *resources.Feature, _ string) (*structvar.StructVar, error) { + if _, err := featureState(input); err != nil { + return nil, err + } + return &structvar.StructVar{Value: input, Refs: nil}, nil } func (r *ResourceFeature) DoRead(ctx context.Context, id string) (*ml.Feature, error) { diff --git a/bundle/direct/dresources/feature_duration.go b/bundle/direct/dresources/feature_duration.go new file mode 100644 index 00000000000..4f9a0d5d46e --- /dev/null +++ b/bundle/direct/dresources/feature_duration.go @@ -0,0 +1,393 @@ +package dresources + +import ( + "encoding/json" + "fmt" + "math" + "regexp" + "strconv" + "strings" + "time" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/dyn/dynvar" + sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// durationComponent matches one leading "" component of a duration string, including +// the day (d) and week (w) units time.ParseDuration lacks. +var durationComponent = regexp.MustCompile(`^([0-9]+(?:\.[0-9]+)?)(ns|us|µs|ms|s|m|h|d|w)`) + +const ( + hoursPerDay = 24 + hoursPerWeek = 24 * 7 + // maxDurationHours is the largest whole-hour count representable as a time.Duration (an int64 + // nanosecond count). + maxDurationHours = int64(math.MaxInt64) / int64(time.Hour) +) + +// duration parses a duration composed of week, day, hour, minute and second units ("1w", "7d", +// "3d15m", "604800s") into the SDK type, which holds only seconds. +func duration(s string) (*sdkduration.Duration, error) { + if parsed, ok := protobufDuration(s); ok { + return parsed, nil + } + d, err := parseExtendedDuration(s) + if err != nil { + return nil, fmt.Errorf("invalid duration %q: %w; use e.g. 604800s, 168h, 7d or 3d15m", s, err) + } + return sdkduration.New(d), nil +} + +// protobufDuration reports whether s is already a protobuf duration, returning the parsed value. +func protobufDuration(s string) (*sdkduration.Duration, bool) { + quoted, err := json.Marshal(s) + if err != nil { + return nil, false + } + var parsed sdkduration.Duration + if err := parsed.UnmarshalJSON(quoted); err != nil { + return nil, false + } + return &parsed, true +} + +// parseExtendedDuration parses a Go duration string extended with day (d, = 24h) and week (w, = +// 168h) units. Components in those units are summed here; all other components (h and below) are +// handed to time.ParseDuration unchanged, which keeps its exact integer handling for the common +// forms. +// +// Duplicates parseTTLDuration in cmd/workspace/postgres/overrides.go, which serves the postgres +// branch --ttl flag. +func parseExtendedDuration(s string) (time.Duration, error) { + // Fast path: no day/week unit means time.ParseDuration handles it all. No other supported unit + // contains 'd' or 'w', so this check is exact. + if !strings.ContainsAny(s, "dw") { + return time.ParseDuration(s) + } + + rest := s + neg := false + switch { + case strings.HasPrefix(rest, "-"): + neg, rest = true, rest[1:] + case strings.HasPrefix(rest, "+"): + rest = rest[1:] + } + + var extra time.Duration // accumulated day/week components + var residual strings.Builder // remaining components, parsed by time.ParseDuration + for rest != "" { + m := durationComponent.FindStringSubmatch(rest) + if m == nil { + return 0, fmt.Errorf("unrecognized duration component %q", rest) + } + switch m[2] { + case "d", "w": + hoursPerUnit := int64(hoursPerDay) + if m[2] == "w" { + hoursPerUnit = hoursPerWeek + } + if strings.Contains(m[1], ".") { + f, err := strconv.ParseFloat(m[1], 64) + if err != nil { + return 0, err + } + ns := f * float64(hoursPerUnit) * float64(time.Hour) + if ns > float64(math.MaxInt64) { + return 0, fmt.Errorf("duration component %s%s overflows", m[1], m[2]) + } + extra += time.Duration(ns) + } else { + n, err := strconv.ParseInt(m[1], 10, 64) + if err != nil { + return 0, err + } + if n > maxDurationHours/hoursPerUnit { + return 0, fmt.Errorf("duration component %s%s overflows", m[1], m[2]) + } + extra += time.Duration(n*hoursPerUnit) * time.Hour + } + default: + residual.WriteString(m[1]) + residual.WriteString(m[2]) + } + rest = rest[len(m[0]):] + } + + total := extra + if residual.Len() > 0 { + d, err := time.ParseDuration(residual.String()) + if err != nil { + return 0, err + } + total += d + } + if neg { + total = -total + } + return total, nil +} + +// optionalDuration converts a duration composed of one or more units (s, h, d, w) into the API type +// that represents duration strictly in seconds, leaving an unset value and an unresolved reference +// alone. +func optionalDuration(s string) (*sdkduration.Duration, error) { + if s == "" || dynvar.IsPureVariableReference(s) { + return nil, nil + } + return duration(s) +} + +// optionalCanonicalDuration converts a duration composed of one or more units (s, h, d, w) into an +// untyped string of the form "604800s", leaving an unset value and an unresolved reference alone. +// The only difference with optionalDuration is the return type. +func optionalCanonicalDuration(s string) (string, error) { + if s == "" || dynvar.IsPureVariableReference(s) { + return s, nil + } + parsed, err := duration(s) + if err != nil { + return "", err + } + marshalled, err := parsed.MarshalJSON() + if err != nil { + return "", err + } + var canonical string + if err := json.Unmarshal(marshalled, &canonical); err != nil { + return "", err + } + return canonical, nil +} + +// durationConversion is one duration field's conversion: the API field name, which names it in the +// error, and the assignment itself. +type durationConversion struct { + field string + apply func() error +} + +// durationConversionForNonStringTypedDuration parses a non-string typed duration. +func durationConversionForNonStringTypedDuration(field, source string, dest **sdkduration.Duration) durationConversion { + return durationConversion{field: field, apply: func() error { + parsed, err := optionalDuration(source) + if err != nil { + return err + } + *dest = parsed + return nil + }} +} + +// durationConversionForStringTypedDuration parses a string typed duration. +func durationConversionForStringTypedDuration(field string, dest *string) durationConversion { + return durationConversion{field: field, apply: func() error { + rewritten, err := optionalCanonicalDuration(*dest) + if err != nil { + return err + } + *dest = rewritten + return nil + }} +} + +// applyDurationConversions walks the registry of duration conversions for each field, applies the +// duration conversion function and reports the field that failed. +func applyDurationConversions(conversions []durationConversion) error { + for _, conversion := range conversions { + if err := conversion.apply(); err != nil { + return fmt.Errorf("%s: %w", conversion.field, err) + } + } + return nil +} + +// fieldWithConvertedDurations copies the API type a config wrapper embeds and converts its +// durations to be strictly in seconds. +// +// input is the config-wrapped field. embedded selects the embedded API type from it, which Go does +// not permit generically: a field on a type parameter is inaccessible. durationConversionsForField +// is that field's registry. +func fieldWithConvertedDurations[Config, State any]( + input *Config, + embedded func(*Config) State, + durationConversionsForField func(*Config, *State) []durationConversion, +) (*State, error) { + if input == nil { + return nil, nil + } + state := embedded(input) + if err := applyDurationConversions(durationConversionsForField(input, &state)); err != nil { + return nil, err + } + return &state, nil +} + +// The registry: every duration field the adapter converts, one entry per field. +// +// When the API gains a duration field, nothing detects it. Add it here by hand: +// +// 1. If the SDK types the field as *sdkduration.Duration, re-type it as a string in +// bundle/config/resources/feature.go so a bundle can write "7d". A field the SDK already types +// as a string needs no config change. +// 2. Add one entry to the durationConversionsFor below that owns the field, picking +// durationConversionForNonStringTypedDuration or durationConversionForStringTypedDuration to +// match the SDK's type for it. +// 3. If the field sits on a message that has no registry yet, write one, then convert that message +// through fieldWithConvertedDurations from whichever conversion owns it: featureState, +// featureFunctionWithConvertedDurations, featureAggregationFunctionWithConvertedDurations, +// featureTimeWindowWithConvertedDurations or featureDataSourceWithConvertedDurations. +// 4. Set the field in featureWithEveryDuration and assert it in +// TestFeatureStateConvertsEveryWindow, which is the only check that the field is converted. + +func durationConversionsForSourceLateness(input *resources.SourceLateness, state *ml.SourceLateness) []durationConversion { + return []durationConversion{ + durationConversionForNonStringTypedDuration("settling_delay", input.SettlingDelay, &state.SettlingDelay), + } +} + +func durationConversionsForRollingWindow(input *resources.RollingWindow, state *ml.RollingWindow) []durationConversion { + return []durationConversion{ + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("window_duration", input.WindowDuration, &state.WindowDuration), + } +} + +func durationConversionsForSawtoothWindow(input *resources.SawtoothWindow, state *ml.SawtoothWindow) []durationConversion { + return []durationConversion{ + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("window_duration", input.WindowDuration, &state.WindowDuration), + } +} + +func durationConversionsForSlidingWindow(input *resources.SlidingWindow, state *ml.SlidingWindow) []durationConversion { + return []durationConversion{ + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("offset", input.Offset, &state.Offset), + durationConversionForStringTypedDuration("window_duration", &state.WindowDuration), + durationConversionForStringTypedDuration("slide_duration", &state.SlideDuration), + } +} + +func durationConversionsForTumblingWindow(input *resources.TumblingWindow, state *ml.TumblingWindow) []durationConversion { + return []durationConversion{ + durationConversionForNonStringTypedDuration("delay", input.Delay, &state.Delay), + durationConversionForNonStringTypedDuration("offset", input.Offset, &state.Offset), + durationConversionForStringTypedDuration("window_duration", &state.WindowDuration), + } +} + +// The config does not shadow a continuous window's durations, so there is nothing to read from it. +func durationConversionsForContinuousWindow(_, state *ml.ContinuousWindow) []durationConversion { + return []durationConversion{ + durationConversionForStringTypedDuration("offset", &state.Offset), + durationConversionForStringTypedDuration("window_duration", &state.WindowDuration), + } +} + +// featureState converts the bundle config for a feature into the API type, parsing every duration. +// The config declares durations as strings; see the override note in +// bundle/config/resources/feature.go. +func featureState(input *resources.Feature) (*ml.Feature, error) { + state := input.Feature + + function, err := featureFunctionWithConvertedDurations(&input.Function) + if err != nil { + return nil, fmt.Errorf("function: %w", err) + } + state.Function = *function + + source, err := featureDataSourceWithConvertedDurations(&input.Source) + if err != nil { + return nil, fmt.Errorf("source: %w", err) + } + state.Source = *source + + // The deprecated top-level time_window is converted on the same terms as the one under the + // aggregation function. + state.TimeWindow, err = featureTimeWindowWithConvertedDurations(input.TimeWindow) + if err != nil { + return nil, fmt.Errorf("time_window: %w", err) + } + + return &state, nil +} + +func featureFunctionWithConvertedDurations(input *resources.Function) (*ml.Function, error) { + state := input.Function + + aggregation, err := featureAggregationFunctionWithConvertedDurations(input.AggregationFunction) + if err != nil { + return nil, fmt.Errorf("aggregation_function: %w", err) + } + state.AggregationFunction = aggregation + + return &state, nil +} + +func featureAggregationFunctionWithConvertedDurations(input *resources.AggregationFunction) (*ml.AggregationFunction, error) { + if input == nil { + return nil, nil + } + state := input.AggregationFunction + + timeWindow, err := featureTimeWindowWithConvertedDurations(input.TimeWindow) + if err != nil { + return nil, fmt.Errorf("time_window: %w", err) + } + state.TimeWindow = timeWindow + + return &state, nil +} + +func featureTimeWindowWithConvertedDurations(input *resources.TimeWindow) (*ml.TimeWindow, error) { + if input == nil { + return nil, nil + } + state := input.TimeWindow + + var err error + if state.Rolling, err = fieldWithConvertedDurations(input.Rolling, + func(c *resources.RollingWindow) ml.RollingWindow { return c.RollingWindow }, + durationConversionsForRollingWindow); err != nil { + return nil, fmt.Errorf("rolling: %w", err) + } + if state.Sawtooth, err = fieldWithConvertedDurations(input.Sawtooth, + func(c *resources.SawtoothWindow) ml.SawtoothWindow { return c.SawtoothWindow }, + durationConversionsForSawtoothWindow); err != nil { + return nil, fmt.Errorf("sawtooth: %w", err) + } + if state.Sliding, err = fieldWithConvertedDurations(input.Sliding, + func(c *resources.SlidingWindow) ml.SlidingWindow { return c.SlidingWindow }, + durationConversionsForSlidingWindow); err != nil { + return nil, fmt.Errorf("sliding: %w", err) + } + if state.Tumbling, err = fieldWithConvertedDurations(input.Tumbling, + func(c *resources.TumblingWindow) ml.TumblingWindow { return c.TumblingWindow }, + durationConversionsForTumblingWindow); err != nil { + return nil, fmt.Errorf("tumbling: %w", err) + } + if state.Continuous, err = fieldWithConvertedDurations(state.Continuous, + func(c *ml.ContinuousWindow) ml.ContinuousWindow { return *c }, + durationConversionsForContinuousWindow); err != nil { + return nil, fmt.Errorf("continuous: %w", err) + } + + return &state, nil +} + +func featureDataSourceWithConvertedDurations(input *resources.DataSource) (*ml.DataSource, error) { + state := input.DataSource + + lateness, err := fieldWithConvertedDurations(input.Lateness, + func(c *resources.SourceLateness) ml.SourceLateness { return c.SourceLateness }, + durationConversionsForSourceLateness) + if err != nil { + return nil, fmt.Errorf("lateness: %w", err) + } + state.Lateness = lateness + + return &state, nil +} diff --git a/bundle/direct/dresources/feature_duration_test.go b/bundle/direct/dresources/feature_duration_test.go new file mode 100644 index 00000000000..cdea98442ff --- /dev/null +++ b/bundle/direct/dresources/feature_duration_test.go @@ -0,0 +1,176 @@ +package dresources + +import ( + "testing" + + "github.com/databricks/cli/bundle/config/resources" + sdkduration "github.com/databricks/databricks-sdk-go/common/types/duration" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseExtendedDuration(t *testing.T) { + for _, tc := range []struct { + input string + want string // canonical protobuf form + wantErr bool + }{ + {input: "604800s", want: "604800s"}, + {input: "168h", want: "604800s"}, + {input: "7d", want: "604800s"}, + {input: "1w", want: "604800s"}, + {input: "3d15m", want: "260100s"}, + {input: "1.5d", want: "129600s"}, + {input: "0.5s", want: "0.500s"}, + {input: "", want: ""}, + {input: "abc", wantErr: true}, + {input: "7x", wantErr: true}, + {input: "7dd", wantErr: true}, + {input: "100000000000w", wantErr: true}, + } { + t.Run(tc.input, func(t *testing.T) { + got, err := optionalCanonicalDuration(tc.input) + if tc.wantErr { + assert.Error(t, err) + return + } + require.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} + +// TestOptionalCanonicalDurationRendersProtobufForm pins that a value the API already accepts is +// still rendered the way the API renders it, so state matches what the server answers with. +func TestOptionalCanonicalDurationRendersProtobufForm(t *testing.T) { + for input, want := range map[string]string{ + "604800s": "604800s", + "0.5s": "0.500s", + "0.500s": "0.500s", + "1.000000001s": "1.000000001s", + "0s": "0s", + } { + got, err := optionalCanonicalDuration(input) + require.NoError(t, err, input) + assert.Equal(t, want, got, input) + } +} + +func seconds(t *testing.T, d *sdkduration.Duration) float64 { + t.Helper() + require.NotNil(t, d) + return d.AsDuration().Seconds() +} + +func TestFeatureStateConvertsEveryWindow(t *testing.T) { + state, err := featureState(featureWithEveryDuration()) + require.NoError(t, err) + + window := state.Function.AggregationFunction.TimeWindow + assert.Equal(t, 86400.0, seconds(t, window.Rolling.WindowDuration)) + assert.Equal(t, 86400.0, seconds(t, window.Rolling.Delay)) + assert.Equal(t, 86400.0, seconds(t, window.Sawtooth.WindowDuration)) + assert.Equal(t, 86400.0, seconds(t, window.Sliding.Offset)) + assert.Equal(t, "86400s", window.Sliding.WindowDuration) + assert.Equal(t, "86400s", window.Sliding.SlideDuration) + assert.Equal(t, "86400s", window.Tumbling.WindowDuration) + assert.Equal(t, "86400s", window.Continuous.WindowDuration) + assert.Equal(t, 86400.0, seconds(t, state.Source.Lateness.SettlingDelay)) + + // The deprecated top-level time_window is converted on the same terms. + assert.Equal(t, 86400.0, seconds(t, state.TimeWindow.Rolling.WindowDuration)) +} + +func TestFeatureStateAcceptsCompoundAndProtobufForms(t *testing.T) { + input := &resources.Feature{} + input.Function.AggregationFunction = &resources.AggregationFunction{ + TimeWindow: &resources.TimeWindow{ + Sawtooth: &resources.SawtoothWindow{WindowDuration: "3d15m"}, + Sliding: &resources.SlidingWindow{}, + }, + } + input.Function.AggregationFunction.TimeWindow.Sliding.SlideDuration = "604800s" + + state, err := featureState(input) + require.NoError(t, err) + window := state.Function.AggregationFunction.TimeWindow + assert.Equal(t, 260100.0, seconds(t, window.Sawtooth.WindowDuration)) + assert.Equal(t, "604800s", window.Sliding.SlideDuration) +} + +func TestFeatureStateLeavesUnsetAndVariableReferences(t *testing.T) { + input := &resources.Feature{} + input.Function.AggregationFunction = &resources.AggregationFunction{ + TimeWindow: &resources.TimeWindow{ + Rolling: &resources.RollingWindow{WindowDuration: "${var.window}"}, + Sliding: &resources.SlidingWindow{}, + }, + } + + state, err := featureState(input) + require.NoError(t, err) + window := state.Function.AggregationFunction.TimeWindow + assert.Nil(t, window.Rolling.WindowDuration, "an unresolved reference is left alone") + assert.Nil(t, window.Rolling.Delay, "an unset duration stays unset") + assert.Empty(t, window.Sliding.WindowDuration) +} + +func TestFeatureStateWithoutWindows(t *testing.T) { + state, err := featureState(&resources.Feature{}) + require.NoError(t, err) + assert.Nil(t, state.Function.AggregationFunction) + assert.Nil(t, state.Source.Lateness) + assert.Nil(t, state.TimeWindow) +} + +func TestFeaturePrepareInputConfigReportsBadDuration(t *testing.T) { + input := &resources.Feature{} + input.Function.AggregationFunction = &resources.AggregationFunction{ + TimeWindow: &resources.TimeWindow{ + Rolling: &resources.RollingWindow{WindowDuration: "7 fortnights"}, + }, + } + + _, err := (*ResourceFeature)(nil).PrepareInputConfig(input, "resources.features.f") + require.Error(t, err) + assert.Contains(t, err.Error(), "function: aggregation_function: time_window: rolling: window_duration:") + assert.Contains(t, err.Error(), `"7 fortnights"`) +} + +func TestFeaturePrepareInputConfigPassesConfigThrough(t *testing.T) { + input := featureWithEveryDuration() + got, err := (*ResourceFeature)(nil).PrepareInputConfig(input, "resources.features.f") + require.NoError(t, err) + assert.Same(t, input, got.Value) + assert.Nil(t, got.Refs) +} + +// featureWithEveryDuration sets every duration field a feature has to one day, so +// TestFeatureStateConvertsEveryWindow can assert each one survives conversion. +func featureWithEveryDuration() *resources.Feature { + const day = "1d" + + window := func() *resources.TimeWindow { + w := &resources.TimeWindow{ + Rolling: &resources.RollingWindow{Delay: day, WindowDuration: day}, + Sawtooth: &resources.SawtoothWindow{Delay: day, WindowDuration: day}, + Sliding: &resources.SlidingWindow{Delay: day, Offset: day}, + Tumbling: &resources.TumblingWindow{Delay: day, Offset: day}, + } + w.Sliding.WindowDuration = day + w.Sliding.SlideDuration = day + w.Tumbling.WindowDuration = day + w.Continuous = &ml.ContinuousWindow{Offset: day, WindowDuration: day} + return w + } + + input := &resources.Feature{ + Function: resources.Function{ + AggregationFunction: &resources.AggregationFunction{TimeWindow: window()}, + }, + Source: resources.DataSource{Lateness: &resources.SourceLateness{SettlingDelay: day}}, + TimeWindow: window(), + } + return input +} diff --git a/bundle/direct/dresources/materialization_set.go b/bundle/direct/dresources/materialization_set.go new file mode 100644 index 00000000000..04dae74d162 --- /dev/null +++ b/bundle/direct/dresources/materialization_set.go @@ -0,0 +1,290 @@ +package dresources + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go" + "github.com/databricks/databricks-sdk-go/apierr" + "github.com/databricks/databricks-sdk-go/common/types/fieldmask" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/google/uuid" +) + +// https://docs.databricks.com/api/workspace/featureengineering +type ResourceMaterializationSet struct { + client *databricks.WorkspaceClient +} + +func (*ResourceMaterializationSet) New(client *databricks.WorkspaceClient) *ResourceMaterializationSet { + return &ResourceMaterializationSet{client: client} +} + +func (*ResourceMaterializationSet) PrepareState(input *resources.MaterializationSet) *ml.MaterializationSet { + return &input.MaterializationSet +} + +// materializationSetUpdatableFields is the only mask the update endpoint accepts. +var materializationSetUpdatableFields = []string{"materialized_features"} + +// escapeMemberKeyPart percent-escapes the separator so the join in materializationSetMemberKey is +// injective. A feature name is validated only for length and non-emptiness, so it may contain the +// separator; two members sharing a key would be diffed as one, and the request would then omit a +// member the server deletes because it cannot see it. +func escapeMemberKeyPart(part string) string { + part = strings.ReplaceAll(part, "%", "%25") + return strings.ReplaceAll(part, "|", "%7C") +} + +// materializationSetMemberKey identifies a materialized feature member by content rather than +// position, as labelled components joined by the separator escapeMemberKeyPart escapes. +// +// online_store_name is omitted rather than left empty when the member has no online destination, +// which is what keeps an offline member distinct from an online one whose store name is unset. +// +// The server pairs members by the same components in MaterializationSetMemberKey -- see +// feature-store/src/main/scala/com/databricks/feature-store/utils/materialized-features/MaterializationSetUtils.scala +// -- holding them in a case class rather than a joined string, so this encoding is the CLI's own +// and only the component list has to agree. Adding, dropping or redefining one takes the same +// change there. +func materializationSetMemberKey(member ml.MaterializedFeature) (string, string) { + parts := []string{"feature_name=" + escapeMemberKeyPart(member.FeatureName)} + if member.OnlineStoreConfig != nil { + parts = append(parts, + "online_store_name="+escapeMemberKeyPart(member.OnlineStoreConfig.OnlineStoreName)) + } + parts = append(parts, "is_streaming_mode="+strconv.FormatBool(member.StreamingMode != nil)) + + return "member", strings.Join(parts, "|") +} + +func (*ResourceMaterializationSet) KeyedSlices() map[string]any { + return map[string]any{ + "materialized_features": materializationSetMemberKey, + } +} + +var ( + materializedFeaturesPattern = structpath.NewPatternDotString(nil, "materialized_features") + materializedFeatureMember = structpath.NewPatternBracketStar(materializedFeaturesPattern) + + // The destination fields the server accepts on write and never returns on read. The whole + // offline block is input-only; online_store_name is returned, so it is deliberately absent. + redactedDestinationPatterns = []*structpath.PatternNode{ + structpath.NewPatternDotString(materializedFeatureMember, "offline_store_config"), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "catalog_name", + ), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "schema_name", + ), + structpath.NewPatternDotString( + structpath.NewPatternDotString(materializedFeatureMember, "online_store_config"), + "table_name_prefix", + ), + } +) + +// isRedactedDestinationField reports whether path names a destination field the read never returns. +func isRedactedDestinationField(path *structpath.PathNode) bool { + for _, pattern := range redactedDestinationPatterns { + if path.HasPatternPrefix(pattern) { + return true + } + } + return false +} + +// isAbsentRemotely reports whether a read returned nothing for a field, which for a redacted field +// is indistinguishable from the server having no value. +func isAbsentRemotely(remote any) bool { + if remote == nil { + return true + } + text, isString := remote.(string) + return isString && text == "" +} + +// OverrideChangeDesc does two things the generic classification gets wrong here: it marks a whole +// materialized feature member added to or removed from the set as an update rather than a +// destroy-and-recreate, and it suppresses the drift left behind by an ignore_remote_changes rule +// that cannot fire. +func (*ResourceMaterializationSet) OverrideChangeDesc(_ context.Context, path *structpath.PathNode, change *ChangeDesc, _ *ml.MaterializationSet) error { + // A member added or removed whole is a two-segment path, materialized_features[member=...] + // carrying the key materializationSetMemberKey builds, and the member list emptied whole is the + // one-segment materialized_features. An immutable field rule + // under materialized_features[*] matches either, because the matcher truncates a pattern longer + // than the path. Both count as an update rather than a destroy-and-recreate, because + // reconciling membership is what the update endpoint does. + if path.Len() <= 2 && path.HasPatternPrefix(materializedFeaturesPattern) { + change.Action = deployplan.Update + return nil + } + + // The ignore_remote_changes suppression on a destination field cannot fire for this resource, + // which keys its member slice on a composite key defined by materializationSetMemberKey. The + // suppression fires only when the persisted value equals the configured + // one, and the persisted-value lookup resolves that composite against the element's field names, + // which never match, so it always returns nothing. An unset change.Old is the surrogate for the + // persisted value being unreachable; a repoint the config asked for carries that value in from + // the local diff instead, and so still recreates. + if isRedactedDestinationField(path) && change.Old == nil && change.New != nil && + isAbsentRemotely(change.Remote) { + change.Action = deployplan.Skip + change.Reason = "input_only_not_returned" + } + return nil +} + +func (r *ResourceMaterializationSet) DoRead(ctx context.Context, id string) (*ml.MaterializationSet, error) { + return r.client.FeatureEngineering.GetMaterializationSet(ctx, ml.GetMaterializationSetRequest{ + MaterializationSetId: id, + }) +} + +// DoCreate submits the set and waits for the long-running operation the create returns. The created +// set arrives as that operation's response, which Wait reaches by polling GetOperation. +func (r *ResourceMaterializationSet) DoCreate(ctx context.Context, config *ml.MaterializationSet) (string, *ml.MaterializationSet, error) { + // The endpoint takes an idempotency token, and a repeated request carrying one already used + // returns the operation the first request with the same idempotency token created. Minted here + // so the token the server sees is the one this call chose. + operation, err := r.client.FeatureEngineering.CreateMaterializationSet(ctx, ml.CreateMaterializationSetRequest{ + MaterializationSet: *config, + RequestId: uuid.NewString(), + }) + if err != nil { + return "", nil, err + } + + set, err := operation.Wait(ctx) + if err != nil { + return "", nil, r.materializationSetOperationFailure(ctx, operation, err) + } + // The id is what the engine records the resource under, so an operation that settles without + // one is a failure even though the wait succeeded. + if set == nil || set.MaterializationSetId == "" { + return "", nil, fmt.Errorf("%s: creation finished without returning the set", + materializationSetOperationTarget(operation)) + } + return set.MaterializationSetId, set, nil +} + +// materializationSetChangeFailures names each materialized feature member a partly applied change +// in the update or delete action could not reach, from whichever progress arm the operation's +// metadata carries. Empty for a creation action, and for a failure (e.g. a validation error) raised +// before the server performed the action. +func materializationSetChangeFailures(metadata *ml.MaterializationSetOperationMetadata) []string { + if metadata == nil { + return nil + } + var members []ml.MaterializedFeatureChangeFailure + switch { + case metadata.UpdateProgress != nil: + members = metadata.UpdateProgress.FailedMaterializedFeatures + case metadata.DeletionProgress != nil: + members = metadata.DeletionProgress.FailedMaterializedFeatures + } + + failures := make([]string, 0, len(members)) + for _, member := range members { + failures = append(failures, fmt.Sprintf("%s (%s): %s", + member.MaterializedFeatureId, member.ErrorCode, member.Message)) + } + return failures +} + +// materializationSetOperationFailure restates a response operation with FAILED status as an API +// error. +// +// StatusCode is set to 200 because the create, update and delete endpoints always return an +// operation with FAILED status when an error is raised, except for the errors raised before the +// action is executed, such as a validation error on the request. +func (r *ResourceMaterializationSet) materializationSetOperationFailure( + ctx context.Context, + operation materializationSetOperation, + waitErr error, +) error { + target := materializationSetOperationTarget(operation) + settled, err := r.client.FeatureEngineering.GetOperation(ctx, ml.GetOperationRequest{ + Name: operation.Name(), + }) + if err != nil || settled.Error == nil { + return fmt.Errorf("%s: %w", target, waitErr) + } + + message := settled.Error.Message + if metadata, err := operation.Metadata(); err == nil { + if failures := materializationSetChangeFailures(metadata); len(failures) > 0 { + message += "; members left unchanged: " + strings.Join(failures, "; ") + } + } + return &apierr.APIError{ + ErrorCode: string(settled.Error.ErrorCode), + Message: target + " operation failed: " + message, + StatusCode: http.StatusOK, + } +} + +// materializationSetOperation is what a create, update and delete operation have in common: the +// three generated interfaces share these two methods and differ only in what Wait returns. +type materializationSetOperation interface { + Name() string + Metadata() (*ml.MaterializationSetOperationMetadata, error) +} + +// materializationSetOperationTarget names the set an operation was changing, taking the set id from +// the operation metadata when the server supplied it and falling back to the operation name. +func materializationSetOperationTarget(operation materializationSetOperation) string { + metadata, err := operation.Metadata() + if err == nil && metadata != nil && metadata.MaterializationSetId != "" { + return "materialization set " + metadata.MaterializationSetId + } + return "operation " + operation.Name() +} + +// DoUpdate reconciles the set's membership and waits for the long-running operation the update +// returns. The reconciled set arrives as that operation's response. +func (r *ResourceMaterializationSet) DoUpdate(ctx context.Context, id string, config *ml.MaterializationSet, _ *PlanEntry) (*ml.MaterializationSet, error) { + update := *config + update.MaterializationSetId = id + + operation, err := r.client.FeatureEngineering.UpdateMaterializationSet(ctx, ml.UpdateMaterializationSetRequest{ + MaterializationSetId: id, + MaterializationSet: update, + UpdateMask: *fieldmask.New(materializationSetUpdatableFields), + RequestId: uuid.NewString(), + }) + if err != nil { + return nil, err + } + + set, err := operation.Wait(ctx) + if err != nil { + return nil, r.materializationSetOperationFailure(ctx, operation, err) + } + return set, nil +} + +// DoDelete deletes the set and waits for the long-running operation the delete returns, which +// settles with no response body. +func (r *ResourceMaterializationSet) DoDelete(ctx context.Context, id string, _ *ml.MaterializationSet) error { + operation, err := r.client.FeatureEngineering.DeleteMaterializationSet(ctx, ml.DeleteMaterializationSetRequest{ + MaterializationSetId: id, + RequestId: uuid.NewString(), + }) + if err != nil { + return err + } + if err := operation.Wait(ctx); err != nil { + return r.materializationSetOperationFailure(ctx, operation, err) + } + return nil +} diff --git a/bundle/direct/dresources/materialization_set_coverage_test.go b/bundle/direct/dresources/materialization_set_coverage_test.go new file mode 100644 index 00000000000..48ba18caccf --- /dev/null +++ b/bundle/direct/dresources/materialization_set_coverage_test.go @@ -0,0 +1,135 @@ +package dresources + +import ( + "reflect" + "strings" + "testing" + + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// setFieldsWithoutRules are the set's own fields that no rule covers: the member slice, whose +// elements are diffed by key, and the id, which the generated config already classifies. +var setFieldsWithoutRules = map[string]bool{ + "materialized_features": true, +} + +// classifiedFields collects the field patterns either lifecycle config mentions, from both the +// hand-written and generated halves, since the planner consults them in sequence. Backend defaults +// are returned apart: one absorbs a field the config leaves unset and says nothing about a value +// the config does declare. +func classifiedFields(t *testing.T) (onDeclaredValue, onlyWhenUndeclared map[string]bool) { + t.Helper() + + handWritten := GetResourceConfig("materialization_sets") + require.NotNil(t, handWritten) + generated := GetGeneratedResourceConfig("materialization_sets") + require.NotNil(t, generated) + + onDeclaredValue = map[string]bool{} + onlyWhenUndeclared = map[string]bool{} + for _, cfg := range []*ResourceLifecycleConfig{handWritten, generated} { + for _, rules := range [][]FieldRule{ + cfg.RecreateOnChanges, + cfg.IgnoreRemoteChanges, + cfg.IgnoreLocalChanges, + cfg.ProvidedIDFields, + cfg.UpdatableIDFields, + cfg.NormalizeSlash, + cfg.StableOutputFields, + } { + for _, rule := range rules { + onDeclaredValue[rule.Field.String()] = true + } + } + for _, rule := range cfg.BackendDefaults { + onlyWhenUndeclared[rule.Field.String()] = true + } + } + return onDeclaredValue, onlyWhenUndeclared +} + +// memberFieldPath builds the path the planner hands to OverrideChangeDesc for one member field. +func memberFieldPath(t *testing.T, memberKey, field string) *structpath.PathNode { + t.Helper() + path, err := structpath.ParsePath( + "materialized_features[member='" + memberKey + "']." + field) + require.NoError(t, err) + return path +} + +// TestMaterializationSetAllMemberFieldsAreClassified keeps the hand-written recreate list +// exhaustive. TestNoUpdateResourcesCoverAllFields stops at HasDoUpdate, and this resource has one, +// so nothing else holds the line: a member field added to the proto later would be classified as a +// plain update, and the endpoint would accept the request and write nothing. +func TestMaterializationSetAllMemberFieldsAreClassified(t *testing.T) { + onDeclaredValue, _ := classifiedFields(t) + + for field := range reflect.TypeFor[ml.MaterializedFeature]().Fields() { + name, ok := jsonFieldName(field) + if !ok { + continue + } + assert.Truef(t, onDeclaredValue["materialized_features[*]."+name], + "member field %q has no rule covering a value the config declares; the set update "+ + "reconciles membership only, so add it to recreate_on_changes. A backend_defaults "+ + "entry does not count: it fires only while the config leaves the field unset", + name, + ) + } +} + +// TestMaterializationSetAllSetFieldsAreClassified does the same for the set's own fields. +func TestMaterializationSetAllSetFieldsAreClassified(t *testing.T) { + onDeclaredValue, onlyWhenUndeclared := classifiedFields(t) + + for field := range reflect.TypeFor[ml.MaterializationSet]().Fields() { + name, ok := jsonFieldName(field) + if !ok || setFieldsWithoutRules[name] { + continue + } + assert.Truef(t, onDeclaredValue[name] || onlyWhenUndeclared[name], + "set field %q is declared in neither materialization_sets.yml nor its generated half", + name, + ) + } +} + +// TestMaterializationSetDestinationFieldsAreSuppressed pins the fields OverrideChangeDesc skips. +// They are input-only, so the read never returns them and the generated rule that would absorb +// the difference cannot fire; a field dropped from this list silently re-plans as a recreate. +func TestMaterializationSetDestinationFieldsAreSuppressed(t *testing.T) { + offline := &ml.MaterializedFeature{ + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "c", SchemaName: "s", TableNamePrefix: "p"}, + } + online := &ml.MaterializedFeature{ + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "store", CatalogName: "c"}, + } + + for _, field := range []string{"catalog_name", "schema_name", "table_name_prefix"} { + _, offlineKey := materializationSetMemberKey(*offline) + _, onlineKey := materializationSetMemberKey(*online) + assert.Truef(t, + isRedactedDestinationField(memberFieldPath(t, offlineKey, "offline_store_config."+field)), + "offline_store_config.%s must be suppressed", field) + assert.Truef(t, + isRedactedDestinationField(memberFieldPath(t, onlineKey, "online_store_config."+field)), + "online_store_config.%s must be suppressed", field) + } + + // online_store_name is returned by the read, so a repoint of it must stay visible. + _, onlineKey := materializationSetMemberKey(*online) + assert.False(t, + isRedactedDestinationField(memberFieldPath(t, onlineKey, "online_store_config.online_store_name"))) +} + +func jsonFieldName(field reflect.StructField) (string, bool) { + tag := field.Tag.Get("json") + if tag == "" || tag == "-" { + return "", false + } + return strings.TrimSuffix(tag, ",omitempty"), true +} diff --git a/bundle/direct/dresources/materialization_set_test.go b/bundle/direct/dresources/materialization_set_test.go new file mode 100644 index 00000000000..318e9e966d7 --- /dev/null +++ b/bundle/direct/dresources/materialization_set_test.go @@ -0,0 +1,216 @@ +package dresources + +import ( + "errors" + "testing" + + "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/libs/structs/structpath" + "github.com/databricks/databricks-sdk-go/service/ml" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMaterializationSetMemberKey(t *testing.T) { + offline := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + } + online := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "store_one"}, + } + streaming := ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + StreamingMode: &ml.StreamingMode{}, + } + + field, value := materializationSetMemberKey(offline) + assert.Equal(t, "member", field) + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=false", value) + + _, onlineValue := materializationSetMemberKey(online) + assert.Equal(t, "feature_name=main.default.feat|online_store_name=store_one|is_streaming_mode=false", onlineValue) + + _, streamingValue := materializationSetMemberKey(streaming) + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=true", streamingValue) + + // The three members of one feature must not collide with each other. + assert.NotEqual(t, value, onlineValue) + assert.NotEqual(t, value, streamingValue) +} + +// A feature name may contain the separator, so without escaping these two distinct members would +// share a key: the plan would diff them as one and the update request would omit the member the +// server then deletes. +func TestMaterializationSetMemberKeyEscapesSeparator(t *testing.T) { + nameHoldsSeparator := ml.MaterializedFeature{ + FeatureName: "main.default.a|b", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "c"}, + } + storeHoldsSeparator := ml.MaterializedFeature{ + FeatureName: "main.default.a", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "b|c"}, + } + + _, first := materializationSetMemberKey(nameHoldsSeparator) + _, second := materializationSetMemberKey(storeHoldsSeparator) + assert.NotEqual(t, first, second) + + // The escape character itself is escaped first, so a name cannot forge an escaped separator. + _, forged := materializationSetMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.a%7Cb", + OnlineStoreConfig: &ml.OnlineStoreConfig{OnlineStoreName: "c"}, + }) + assert.NotEqual(t, first, forged) +} + +// An absent online destination and an online destination carrying no store name are distinct +// members, which is why the component is omitted rather than left empty. +func TestMaterializationSetMemberKeySeparatesAbsentOnlineStoreFromEmptyName(t *testing.T) { + _, offline := materializationSetMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OfflineStoreConfig: &ml.OfflineStoreConfig{CatalogName: "main", SchemaName: "default"}, + }) + _, onlineWithoutName := materializationSetMemberKey(ml.MaterializedFeature{ + FeatureName: "main.default.feat", + OnlineStoreConfig: &ml.OnlineStoreConfig{CatalogName: "main", SchemaName: "default"}, + }) + + assert.Equal(t, "feature_name=main.default.feat|is_streaming_mode=false", offline) + assert.Equal(t, + "feature_name=main.default.feat|online_store_name=|is_streaming_mode=false", + onlineWithoutName) + assert.NotEqual(t, offline, onlineWithoutName) +} + +// operationStub stands in for a generated operation interface. The three verbs' interfaces differ +// only in what Wait returns, which materializationSetOperationTarget never calls. +type operationStub struct { + name string + metadata *ml.MaterializationSetOperationMetadata + err error +} + +func (s operationStub) Name() string { return s.name } + +func (s operationStub) Metadata() (*ml.MaterializationSetOperationMetadata, error) { + return s.metadata, s.err +} + +// The set id names what a failed create, update or delete was changing. The server assigns it +// before reserving the operation record, so the fallback to the operation name covers the cases +// where the metadata is unreadable rather than a set that has no id. +func TestMaterializationSetOperationTarget(t *testing.T) { + assert.Equal(t, "materialization set set-1", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + metadata: &ml.MaterializationSetOperationMetadata{MaterializationSetId: "set-1"}, + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + metadata: &ml.MaterializationSetOperationMetadata{}, + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + })) + + assert.Equal(t, "operation operations/abc", materializationSetOperationTarget(operationStub{ + name: "operations/abc", + err: errors.New("metadata is not a MaterializationSetOperationMetadata"), + })) +} + +func TestMaterializationSetChangeFailuresFromUpdateProgress(t *testing.T) { + failures := materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + UpdateProgress: &ml.MaterializationSetUpdateProgress{ + DeletedMaterializedFeatureIds: []string{"mf-gone"}, + FailedMaterializedFeatures: []ml.MaterializedFeatureChangeFailure{ + { + MaterializedFeatureId: "mf-kept", + ErrorCode: "PERMISSION_DENIED", + Message: "no permission on the online table", + }, + }, + }, + }) + + assert.Equal(t, []string{"mf-kept (PERMISSION_DENIED): no permission on the online table"}, failures) +} + +func TestMaterializationSetChangeFailuresFromDeletionProgress(t *testing.T) { + failures := materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + DeletionProgress: &ml.MaterializationSetDeletionProgress{ + FailedMaterializedFeatures: []ml.MaterializedFeatureChangeFailure{ + {MaterializedFeatureId: "mf-1", ErrorCode: "INTERNAL_ERROR", Message: "pipeline stuck"}, + {MaterializedFeatureId: "mf-2", ErrorCode: "RESOURCE_CONFLICT", Message: "in use"}, + }, + }, + }) + + assert.Equal(t, []string{ + "mf-1 (INTERNAL_ERROR): pipeline stuck", + "mf-2 (RESOURCE_CONFLICT): in use", + }, failures) +} + +func TestMaterializationSetChangeFailuresWithoutFailedMembers(t *testing.T) { + // A creation carries no progress arm at all, and a change that fully applied carries one + // naming no failure; neither has a member to report. + assert.Empty(t, materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{})) + assert.Empty(t, materializationSetChangeFailures(&ml.MaterializationSetOperationMetadata{ + DeletionProgress: &ml.MaterializationSetDeletionProgress{ + DeletedMaterializedFeatureIds: []string{"mf-1"}, + }, + })) + assert.Empty(t, materializationSetChangeFailures(nil)) +} + +// TestMaterializationSetMembershipChangeIsAnUpdate enumerates every path shape the planner hands +// this resource, so the membership carve-out's reach is visible rather than inferred. The two +// shapes it covers are a member added or removed whole and the member list emptied whole; every +// other path keeps the action the rules gave it. +func TestMaterializationSetMembershipChangeIsAnUpdate(t *testing.T) { + const member = "materialized_features[member='feature_name=main.ns.feat|online_store_name=store|is_streaming_mode=false']" + + for _, test := range []struct { + path string + action deployplan.ActionType + }{ + {"materialized_features", deployplan.Update}, + {member, deployplan.Update}, + {"materialized_features[member='feature_name=main.ns.feat|is_streaming_mode=false']", deployplan.Update}, + {member + ".feature_name", deployplan.Recreate}, + {member + ".cron_schedule_trigger", deployplan.Recreate}, + {member + ".online_store_config.table_name_prefix", deployplan.Recreate}, + {"materialization_set_id", deployplan.Recreate}, + } { + t.Run(test.path, func(t *testing.T) { + path, err := structpath.ParsePath(test.path) + require.NoError(t, err) + + change := &ChangeDesc{Action: deployplan.Recreate, Reason: "immutable"} + require.NoError(t, (&ResourceMaterializationSet{}).OverrideChangeDesc( + t.Context(), path, change, nil)) + assert.Equal(t, test.action, change.Action) + }) + } +} + +// TestMaterializationSetDestinationSuppressionStillReachesMemberFields keeps the two blocks of +// OverrideChangeDesc disjoint: widening the membership carve-out to one segment must not shadow the +// input-only suppression, which only ever sees a member's destination three segments down. +func TestMaterializationSetDestinationSuppressionStillReachesMemberFields(t *testing.T) { + path, err := structpath.ParsePath( + "materialized_features[member='feature_name=main.ns.feat|is_streaming_mode=false'].offline_store_config") + require.NoError(t, err) + + change := &ChangeDesc{Action: deployplan.Recreate, Reason: "spec:immutable", New: "declared"} + require.NoError(t, (&ResourceMaterializationSet{}).OverrideChangeDesc( + t.Context(), path, change, nil)) + + assert.Equal(t, deployplan.Skip, change.Action) + assert.Equal(t, "input_only_not_returned", change.Reason) +} diff --git a/bundle/direct/dresources/serialize_test.go b/bundle/direct/dresources/serialize_test.go index 76078460193..a7827216a6f 100644 --- a/bundle/direct/dresources/serialize_test.go +++ b/bundle/direct/dresources/serialize_test.go @@ -145,8 +145,14 @@ func TestRoundtripAllFieldsRemoteType(t *testing.T) { // marshal.Unmarshal hands the whole payload to the embedded member, so the shadowed // string comes back holding the raw JSON text and reads as a lost field. Its other // fields go unchecked on this surface as a result. +// +// features is exempt for the same reason. Its window types declare their durations as strings and +// deliberately shadow the same-named duration fields of the embedded API types, so a bundle can +// write "7d" (featureState parses it to the protobuf form before deploy). Only the shallower string +// is reachable by that JSON name, so the shadowed duration cannot round-trip, and marshal.Unmarshal +// leaves the embedded members' other fields unset. Its other fields go unchecked here as a result. func TestRoundtripAllFieldsInputConfigType(t *testing.T) { - testRoundtripAllFields(t, "InputConfigType", (*Adapter).InputConfigType, []string{"cluster_policies"}) + testRoundtripAllFields(t, "InputConfigType", (*Adapter).InputConfigType, []string{"cluster_policies", "features"}) } var jsonMarshalerType = reflect.TypeFor[json.Marshaler]() diff --git a/bundle/direct/keyed_member_pattern_match_test.go b/bundle/direct/keyed_member_pattern_match_test.go new file mode 100644 index 00000000000..ac5e15f1a92 --- /dev/null +++ b/bundle/direct/keyed_member_pattern_match_test.go @@ -0,0 +1,29 @@ +package direct + +import ( + "testing" + + "github.com/databricks/cli/libs/structs/structpath" + "github.com/stretchr/testify/assert" +) + +// A whole-member change on a keyed slice is a two-segment path, so it matches any deeper +// per-field pattern through the short-path branch of matchesFieldRuleBidirectional. A +// per-field recreate rule therefore recreates the parent when a member is merely added or +// removed, which is why a keyed-slice resource needs a carve-out before classification. +func TestKeyedMemberChangeMatchesPerFieldPattern(t *testing.T) { + // configs/.yml: materialized_features[*].feature_name + pattern := structpath.NewPatternDotString(nil, "materialized_features") + pattern = structpath.NewPatternBracketStar(pattern) + perField := structpath.NewPatternDotString(pattern, "feature_name") + + // structdiff emits this for a member present on one side only. + member := structpath.NewDotString(nil, "materialized_features") + member = structpath.NewKeyValue(member, "feature_name", "main.ns.feat") + + assert.Equal(t, 3, perField.Len()) + assert.Equal(t, 2, member.Len()) + assert.False(t, member.HasPatternPrefix(perField), "member is shorter, so not a direct prefix match") + assert.True(t, matchesFieldRuleBidirectional(member, perField), + "whole-member change still matches the per-field rule via the truncated pattern") +} diff --git a/bundle/phases/deploy.go b/bundle/phases/deploy.go index 13c5121f569..d7206c68607 100644 --- a/bundle/phases/deploy.go +++ b/bundle/phases/deploy.go @@ -41,6 +41,7 @@ var deployApprovalGroups = []approvalGroup{ {group: "postgres_databases", message: deleteOrRecreatePostgresDatabaseMessage}, {group: "vector_search_indexes", message: deleteOrRecreateVectorSearchIndexMessage}, {group: "genie_spaces", message: deleteOrRecreateGenieSpaceMessage}, + {group: "materialization_sets", message: deleteOrRecreateMaterializationSetMessage}, } func approvalForDeploy(ctx context.Context, b *bundle.Bundle, plan *deployplan.Plan) (bool, error) { diff --git a/bundle/phases/messages.go b/bundle/phases/messages.go index 03eba35d8cf..bca18c53ab7 100644 --- a/bundle/phases/messages.go +++ b/bundle/phases/messages.go @@ -49,6 +49,11 @@ index drops all upserted vectors. Both can be expensive to rebuild:` deleteOrRecreateGenieSpaceMessage = ` This action will result in the deletion or recreation of the following Genie spaces. The conversation history attached to a deleted space is permanently lost:` + + deleteOrRecreateMaterializationSetMessage = ` +This action will result in the deletion or recreation of the following materialization sets. +Every materialized feature in a recreated set is dropped and recomputed from scratch, including +the members whose configuration did not change:` ) // DataLossWarning is the warning shown when a non-interactive command is about diff --git a/bundle/statemgmt/state_load_test.go b/bundle/statemgmt/state_load_test.go index 4ec77578d2b..d55633c14ff 100644 --- a/bundle/statemgmt/state_load_test.go +++ b/bundle/statemgmt/state_load_test.go @@ -32,6 +32,7 @@ func TestStateToBundleEmptyLocalResources(t *testing.T) { "resources.models.test_mlflow_model": {ID: "1"}, "resources.experiments.test_mlflow_experiment": {ID: "1"}, "resources.features.test_feature": {ID: "main.default.test_feature"}, + "resources.materialization_sets.test_materialization_set": {ID: "set1"}, "resources.model_serving_endpoints.test_model_serving": {ID: "1"}, "resources.model_services.test_model_service": {ID: "main.default.test_model_service"}, "resources.mcp_services.test_mcp_service": {ID: "main.default.test_mcp_service"}, @@ -225,6 +226,15 @@ func TestStateToBundleEmptyRemoteResources(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "test_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.default.test_feature"}, + }, + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ @@ -664,6 +674,15 @@ func TestStateToBundleModifiedResources(t *testing.T) { }, }, }, + MaterializationSets: map[string]*resources.MaterializationSet{ + "test_materialization_set": { + MaterializationSet: ml.MaterializationSet{ + MaterializedFeatures: []ml.MaterializedFeature{ + {FeatureName: "main.default.test_feature"}, + }, + }, + }, + }, ModelServingEndpoints: map[string]*resources.ModelServingEndpoint{ "test_model_serving": { CreateServingEndpoint: serving.CreateServingEndpoint{ diff --git a/libs/testserver/fake_workspace.go b/libs/testserver/fake_workspace.go index 78234c5ebfa..a2246db4023 100644 --- a/libs/testserver/fake_workspace.go +++ b/libs/testserver/fake_workspace.go @@ -229,6 +229,8 @@ type FakeWorkspace struct { ModelProviderServices map[string]catalog.ModelProviderService Features map[string]ml.Feature + MaterializationSets map[string]ml.MaterializationSet + Operations map[string]ml.Operation ServingEndpoints map[string]serving.ServingEndpointDetailed VectorSearchEndpoints map[string]vectorsearch.EndpointInfo VectorSearchIndexes map[string]fakeVectorSearchIndex @@ -513,6 +515,8 @@ func NewFakeWorkspace(url, token string) *FakeWorkspace { McpServices: map[string]catalog.McpService{}, ModelProviderServices: map[string]catalog.ModelProviderService{}, Features: map[string]ml.Feature{}, + MaterializationSets: map[string]ml.MaterializationSet{}, + Operations: map[string]ml.Operation{}, Volumes: map[string]catalog.VolumeInfo{}, Dashboards: NewEventualMap[string, *fakeDashboard](eventualConsistency), PublishedDashboards: map[string]dashboards.PublishedDashboard{}, diff --git a/libs/testserver/handlers.go b/libs/testserver/handlers.go index e1a14b9800b..f93e38067bb 100644 --- a/libs/testserver/handlers.go +++ b/libs/testserver/handlers.go @@ -851,6 +851,23 @@ func AddDefaultHandlers(server *Server) { return MapDelete(req.Workspace, req.Workspace.Features, req.Vars["full_name"]) }) + // Feature Engineering materialization sets: + server.Handle("POST", "/api/2.0/feature-engineering/materialization-sets", func(req Request) any { + return req.Workspace.MaterializationSetsCreate(req) + }) + server.Handle("GET", "/api/2.0/feature-engineering/operations/{operation_id}", func(req Request) any { + return req.Workspace.OperationsGet(req.Vars["operation_id"]) + }) + server.Handle("GET", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return MapGet(req.Workspace, req.Workspace.MaterializationSets, req.Vars["materialization_set_id"]) + }) + server.Handle("PATCH", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return req.Workspace.MaterializationSetsUpdate(req, req.Vars["materialization_set_id"]) + }) + server.Handle("DELETE", "/api/2.0/feature-engineering/materialization-sets/{materialization_set_id}", func(req Request) any { + return req.Workspace.MaterializationSetsDelete(req.Vars["materialization_set_id"]) + }) + // Model Provider Services (AI Gateway): server.Handle("POST", "/api/2.1/unity-catalog/model-provider-services", func(req Request) any { diff --git a/libs/testserver/materialization_set.go b/libs/testserver/materialization_set.go new file mode 100644 index 00000000000..3eb19af1988 --- /dev/null +++ b/libs/testserver/materialization_set.go @@ -0,0 +1,765 @@ +package testserver + +import ( + "encoding/json" + "fmt" + "net/http" + "sort" + "strings" + + "github.com/databricks/databricks-sdk-go/service/ml" +) + +// This fake mirrors UpdateMaterializationSet closely enough that one set of acceptance goldens +// serves both the local run and a run against a real workspace. Where it rejects, it returns the +// backend's own message text, so a golden recorded locally is a claim about the real API that the +// cloud run then checks. + +// materializationSetRefusal is a rule's verdict, separated from how it is reported: a rule the +// backend reaches before reserving an operation record becomes an HTTP error, and one it reaches +// after becomes a settled FAILED operation. +type materializationSetRefusal struct { + errorCode string + message string +} + +func materializationSetError(statusCode int, errorCode, message string) Response { + return Response{ + StatusCode: statusCode, + Body: map[string]string{ + "error_code": errorCode, + "message": message, + }, + } +} + +// quotedMaterializedFeatureIDs renders an id list the way FeatureStoreMessages.quotedIds does. +func quotedMaterializedFeatureIDs(ids []string) string { + quoted := make([]string, len(ids)) + for i, id := range ids { + quoted[i] = "'" + id + "'" + } + return strings.Join(quoted, ", ") +} + +// materializedFeatureFieldCannotBeChanged refuses a field the set update cannot write. No +// acceptance case reaches it: the planner classifies every member field as a recreate, so a +// declared change never becomes an update request. +func materializedFeatureFieldCannotBeChanged( + featureName, materializedFeatureID, fieldPath string, +) Response { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("The fields '%s' of materialized feature '%s' of feature '%s' cannot be "+ + "changed. Only removing members is supported; recreate the materialization set "+ + "instead.", fieldPath, materializedFeatureID, featureName)) +} + +// materializationSetMemberKey matches a requested member to a stored one the way the backend does: +// by feature name, online store and whether the trigger is streaming. Feature name alone is not +// unique because one feature has an offline member plus one member per online store. +func materializationSetMemberKey(member ml.MaterializedFeature) string { + onlineStoreName := "" + if member.OnlineStoreConfig != nil { + onlineStoreName = member.OnlineStoreConfig.OnlineStoreName + } + return fmt.Sprintf("%s|%s|%t", member.FeatureName, onlineStoreName, member.StreamingMode != nil) +} + +// effectivePipelineScheduleState resolves the create-path default, so a request mixing unset and +// ACTIVE members is not read as a disagreement. +func effectivePipelineScheduleState( + state ml.MaterializedFeaturePipelineScheduleState, +) ml.MaterializedFeaturePipelineScheduleState { + if state == "" { + return ml.MaterializedFeaturePipelineScheduleStateActive + } + return state +} + +// populateMaterializationSetMember fills the fields the backend assigns on create, then redacts +// the INPUT_ONLY destination fields the way the response path does. Storing them redacted is what +// makes a read here match a read from a real workspace, which an echoing fake hides. +func populateMaterializationSetMember(member ml.MaterializedFeature, setID string) ml.MaterializedFeature { + member.MaterializedFeatureId = nextUUID() + member.MaterializationSetId = setID + member.IsOnline = member.OnlineStoreConfig != nil + member.PipelineScheduleState = effectivePipelineScheduleState(member.PipelineScheduleState) + member.OfflineStoreConfig = nil + if member.OnlineStoreConfig != nil { + member.OnlineStoreConfig = &ml.OnlineStoreConfig{ + OnlineStoreName: member.OnlineStoreConfig.OnlineStoreName, + } + } + return member +} + +// validateBatchPipelineScheduleStates mirrors the create-path rule that every batch member agrees +// on one effective schedule state. Streaming members never carry one, so they are excluded. +func validateBatchPipelineScheduleStates(members []ml.MaterializedFeature) *Response { + states := map[ml.MaterializedFeaturePipelineScheduleState]bool{} + for _, member := range members { + if member.StreamingMode != nil { + continue + } + states[effectivePipelineScheduleState(member.PipelineScheduleState)] = true + } + if len(states) > 1 { + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "All materialized features in a BatchCreateMaterializedFeaturesRequest call must "+ + "have the same pipeline schedule state.") + return &response + } + return nil +} + +// validateSingleCronScheduleMode mirrors the create-path rule that one request carries one cron +// schedule mode. A sawtooth feature must be DERIVED and a batch-only feature MANUAL, so this is +// what a set holding both runs into first. +func validateSingleCronScheduleMode(members []ml.MaterializedFeature) *Response { + modes := map[ml.CronScheduleMode]bool{} + for _, member := range members { + if member.CronScheduleTrigger == nil { + continue + } + modes[member.CronScheduleTrigger.Mode] = true + } + if len(modes) > 1 { + response := materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "All cron-scheduled features in a single request must use the same cron schedule "+ + "mode; a request cannot mix DERIVED (service-derived) and MANUAL "+ + "(caller-supplied) schedules.") + return &response + } + return nil +} + +// featureMaterializationDestination renders the destination pair the create compares across +// features: a feature's offline config plus the online config it is also materialized to, if any. +func featureMaterializationDestination(members []ml.MaterializedFeature) string { + offline, online := "", "" + for _, member := range members { + if cfg := member.OfflineStoreConfig; cfg != nil { + offline = cfg.CatalogName + "." + cfg.SchemaName + "." + cfg.TableNamePrefix + } + if cfg := member.OnlineStoreConfig; cfg != nil { + online = cfg.OnlineStoreName + ":" + cfg.CatalogName + "." + cfg.SchemaName + "." + + cfg.TableNamePrefix + } + } + return offline + "|" + online +} + +// validateSharedMaterializationDestination mirrors the create-path rule that every feature in a +// set asks for an identical destination pair. Because the pair carries the online config, a set +// cannot mix a feature materialized to both stores with one materialized offline only. +func validateSharedMaterializationDestination(members []ml.MaterializedFeature) *materializationSetRefusal { + membersByFeature := map[string][]ml.MaterializedFeature{} + featureOrder := []string{} + for _, member := range members { + if _, seen := membersByFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + membersByFeature[member.FeatureName] = append(membersByFeature[member.FeatureName], member) + } + + destinations := map[string]bool{} + for _, featureName := range featureOrder { + destinations[featureMaterializationDestination(membersByFeature[featureName])] = true + } + if len(destinations) > 1 { + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: "All features must share the same materialization destination.", + } + } + return validateOfflineAndOnlineDestinationsDiffer(members) +} + +// validateOfflineAndOnlineDestinationsDiffer mirrors the create-path rule that a feature +// materialized to both stores must separate them by catalog, schema or prefix. +func validateOfflineAndOnlineDestinationsDiffer(members []ml.MaterializedFeature) *materializationSetRefusal { + var offline *ml.OfflineStoreConfig + var online *ml.OnlineStoreConfig + for _, member := range members { + if member.OfflineStoreConfig != nil { + offline = member.OfflineStoreConfig + } + if member.OnlineStoreConfig != nil { + online = member.OnlineStoreConfig + } + } + if offline == nil || online == nil { + return nil + } + if online.CatalogName != offline.CatalogName || online.SchemaName != offline.SchemaName || + online.TableNamePrefix != offline.TableNamePrefix { + return nil + } + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf("Offline and online materialization destinations must differ. Found "+ + "identical catalog names '%s', schema names '%s', and table name prefixes '%s'.", + offline.CatalogName, offline.SchemaName, offline.TableNamePrefix), + } +} + +// sawtoothMaterializedFeatureCount is how many materialized features a sawtooth window produces: +// an offline batch one, an online batch one and an online streaming one. +const sawtoothMaterializedFeatureCount = 3 + +// isSawtoothFeature reports whether a stored feature aggregates over a sawtooth window. An unknown +// feature is not treated as sawtooth: the real create resolves the feature first and fails on a +// missing one, which is a separate rejection this fake does not model. +func (s *FakeWorkspace) isSawtoothFeature(featureName string) bool { + feature, stored := s.Features[featureName] + if !stored { + return false + } + aggregation := feature.Function.AggregationFunction + return aggregation != nil && aggregation.TimeWindow != nil && aggregation.TimeWindow.Sawtooth != nil +} + +// validateBatchAndStreamingFeaturesNotMixed mirrors the create-path rule that one call materializes +// either batch features or streaming features. A Stream-sourced feature always materializes on the +// streaming path; a Delta-sourced one joins it only when a member of it carries a streaming +// trigger. An unknown feature falls in neither group, for the reason isSawtoothFeature gives. +func (s *FakeWorkspace) validateBatchAndStreamingFeaturesNotMixed( + members []ml.MaterializedFeature, +) *materializationSetRefusal { + var batchFeatureNames, streamingFeatureNames []string + for _, featureName := range materializationSetFeatureNames(members) { + feature, stored := s.Features[featureName] + if !stored { + continue + } + hasStreamingMember := false + for _, member := range members { + if member.FeatureName == featureName && member.StreamingMode != nil { + hasStreamingMember = true + } + } + switch { + case feature.Source.StreamSource != nil || hasStreamingMember: + streamingFeatureNames = append(streamingFeatureNames, featureName) + case feature.Source.DeltaTableSource != nil: + batchFeatureNames = append(batchFeatureNames, featureName) + } + } + if len(batchFeatureNames) == 0 || len(streamingFeatureNames) == 0 { + return nil + } + + return &materializationSetRefusal{ + errorCode: "INVALID_PARAMETER_VALUE", + message: fmt.Sprintf("Batch and streaming features cannot be mixed in a single "+ + "materialize_features call because they require different trigger types. Issue two "+ + "separate calls: one for batch features and one for streaming features. Batch "+ + "features: %s. Streaming features: %s", + strings.Join(batchFeatureNames, ", "), strings.Join(streamingFeatureNames, ", ")), + } +} + +// isDerivedCronSchedule mirrors MaterializedFeatureUtils.isDerivedCronSchedule: only a cron +// schedule trigger carrying DERIVED opts out of a pinned schedule, and an unset mode reads as +// MANUAL. +func isDerivedCronSchedule(member ml.MaterializedFeature) bool { + return member.CronScheduleTrigger != nil && + member.CronScheduleTrigger.Mode == ml.CronScheduleModeDerived +} + +// validateSawtoothMaterializedFeatures mirrors the rule that a sawtooth feature declares all three +// of its materialized features itself. The request is not expanded for it: a materialization set +// stamps a set id on every member before the backend reaches its normalization, and that +// normalization is skipped whenever the id is set, so a member maps to exactly one materialized +// feature and the set stays diffable. +func (s *FakeWorkspace) validateSawtoothMaterializedFeatures( + members []ml.MaterializedFeature, +) *materializationSetRefusal { + byFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if !s.isSawtoothFeature(member.FeatureName) { + continue + } + if _, seen := byFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + byFeature[member.FeatureName] = append(byFeature[member.FeatureName], member) + } + + for _, featureName := range featureOrder { + var offlineBatch, onlineBatch, onlineStreaming int + for _, member := range byFeature[featureName] { + switch { + case member.OnlineStoreConfig != nil && member.StreamingMode != nil: + onlineStreaming++ + case member.OnlineStoreConfig != nil: + onlineBatch++ + case member.OfflineStoreConfig != nil && member.StreamingMode == nil: + offlineBatch++ + } + } + if len(byFeature[featureName]) != sawtoothMaterializedFeatureCount || + offlineBatch != 1 || onlineBatch != 1 || onlineStreaming != 1 { + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf( + "Feature '%s' uses a sawtooth window, which requires an offline batch "+ + "materialization, an online batch materialization and an online streaming "+ + "materialization. Declare one materialize feature request for each.", + featureName), + } + } + + for _, member := range byFeature[featureName] { + if member.StreamingMode != nil || isDerivedCronSchedule(member) { + continue + } + return &materializationSetRefusal{ + errorCode: "BAD_REQUEST", + message: fmt.Sprintf( + "Feature '%s' must declare a DERIVED cron schedule on its batch materialize "+ + "feature requests. The sawtooth batch materializations refresh on a daily "+ + "schedule the service computes, so a caller neither pins nor omits it.", + featureName), + } + } + } + return nil +} + +// MaterializationSetsCreate fakes POST /api/2.0/feature-engineering/materialization-sets. +func (s *FakeWorkspace) MaterializationSetsCreate(req Request) Response { + defer s.LockUnlock()() + + var set ml.MaterializationSet + if err := json.Unmarshal(req.Body, &set); err != nil { + return materializationSetInternalError(err) + } + + if len(set.MaterializedFeatures) == 0 { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + "A materialization set requires at least one materialized feature.") + } + if bad := validateBatchPipelineScheduleStates(set.MaterializedFeatures); bad != nil { + return *bad + } + if bad := validateSingleCronScheduleMode(set.MaterializedFeatures); bad != nil { + return *bad + } + set.MaterializationSetId = nextUUID() + + // The operation record is reserved here, so what follows is the operation's outcome rather + // than a rejected request: the backend reaches these rules inside the batch create it calls + // after reserving, and reports them by settling the record. + for _, bad := range []*materializationSetRefusal{ + s.validateBatchAndStreamingFeaturesNotMixed(set.MaterializedFeatures), + validateSharedMaterializationDestination(set.MaterializedFeatures), + s.validateSawtoothMaterializedFeatures(set.MaterializedFeatures), + } { + if bad == nil { + continue + } + return s.materializationSetFailedOperationResponse(set.MaterializationSetId, + set.MaterializedFeatures, bad.errorCode, bad.message, nil) + } + for i, member := range set.MaterializedFeatures { + set.MaterializedFeatures[i] = populateMaterializationSetMember(member, set.MaterializationSetId) + } + + s.MaterializationSets[set.MaterializationSetId] = set + + return s.materializationSetOperationResponse(set.MaterializationSetId, set.MaterializedFeatures, set) +} + +// materializationSetOperationResponse records the terminal operation a set change returns and +// renders it as the endpoint's response. The work is finished by the time the response is written, +// so the operation is recorded already terminal; a caller still has to poll it, because a create's +// or update's resulting set is reachable only from there. +func (s *FakeWorkspace) materializationSetOperationResponse( + setID string, + members []ml.MaterializedFeature, + result any, +) Response { + metadata, err := json.Marshal(ml.MaterializationSetOperationMetadata{ + FeatureFullNames: materializationSetFeatureNames(members), + MaterializationSetId: setID, + State: ml.MaterializationSetOperationMetadataStateSucceeded, + }) + if err != nil { + return materializationSetInternalError(err) + } + response, err := json.Marshal(result) + if err != nil { + return materializationSetInternalError(err) + } + + operation := ml.Operation{ + Name: "operations/" + nextUUID(), + Done: true, + Metadata: metadata, + Response: response, + } + s.Operations[operation.Name] = operation + return Response{Body: operation} +} + +// materializedFeatureTeardownFailsMarker is how a fixture asks the fake for a member the backend +// cannot tear down. A real teardown fails on infrastructure -- a job, a pipeline or a table the +// delete cannot reach -- which no request can ask for, so the trigger has to be in the fixture. +const materializedFeatureTeardownFailsMarker = "teardown_fails" + +// materializationSetTeardownFailures reports the members of `targets` the fake refuses to tear +// down, one failure each, in target order. +func materializationSetTeardownFailures( + targets [][]ml.MaterializedFeature, + removedIDs map[string]bool, +) []ml.MaterializedFeatureChangeFailure { + var failures []ml.MaterializedFeatureChangeFailure + for _, target := range targets { + if !strings.Contains(target[0].FeatureName, materializedFeatureTeardownFailsMarker) { + continue + } + for _, member := range target { + if !removedIDs[member.MaterializedFeatureId] { + continue + } + failures = append(failures, ml.MaterializedFeatureChangeFailure{ + MaterializedFeatureId: member.MaterializedFeatureId, + ErrorCode: "INTERNAL_ERROR", + Message: fmt.Sprintf( + "Failed to delete materialized feature '%s'. If the issue persists, please "+ + "retry or contact support.", member.MaterializedFeatureId), + }) + } + } + return failures +} + +// partialMaterializationSetChangeErrorCode collapses the member failures to one code the way the +// backend does: a single code is reported as itself, a permission failure wins over any other, and +// a mix of client errors becomes BAD_REQUEST. +func partialMaterializationSetChangeErrorCode( + failures []ml.MaterializedFeatureChangeFailure, +) string { + codes := map[string]bool{} + for _, failure := range failures { + codes[failure.ErrorCode] = true + } + switch { + case len(codes) == 1: + for code := range codes { + return code + } + return "INTERNAL_ERROR" + case codes["PERMISSION_DENIED"]: + return "PERMISSION_DENIED" + default: + return "INTERNAL_ERROR" + } +} + +// materializationSetFailedOperationResponse settles an operation as FAILED and returns it with +// HTTP 200, which is how the backend reports a change it reserved an operation for and then could +// not finish. The per-member outcome rides on the metadata's progress arm. +func (s *FakeWorkspace) materializationSetFailedOperationResponse( + setID string, + members []ml.MaterializedFeature, + errorCode string, + message string, + progress func(*ml.MaterializationSetOperationMetadata), +) Response { + settled := ml.MaterializationSetOperationMetadata{ + FeatureFullNames: materializationSetFeatureNames(members), + MaterializationSetId: setID, + State: ml.MaterializationSetOperationMetadataStateFailed, + } + if progress != nil { + progress(&settled) + } + + metadata, err := json.Marshal(settled) + if err != nil { + return materializationSetInternalError(err) + } + + operation := ml.Operation{ + Name: "operations/" + nextUUID(), + Done: true, + Metadata: metadata, + Error: &ml.DatabricksServiceExceptionWithDetailsProto{ + ErrorCode: ml.ErrorCode(errorCode), + Message: message, + }, + } + s.Operations[operation.Name] = operation + return Response{Body: operation} +} + +// materializationSetFeatureNames lists each member's feature once in member order, which is what +// the operation metadata reports. +func materializationSetFeatureNames(members []ml.MaterializedFeature) []string { + seen := map[string]bool{} + var names []string + for _, member := range members { + if !seen[member.FeatureName] { + seen[member.FeatureName] = true + names = append(names, member.FeatureName) + } + } + return names +} + +func materializationSetInternalError(err error) Response { + return Response{ + Body: fmt.Sprintf("internal error: %s", err), + StatusCode: http.StatusInternalServerError, + } +} + +// materializationSetNotFound refuses a set id no set holds. +func materializationSetNotFound(setID string) Response { + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Materialization set with ID '%s' does not exist.", setID)) +} + +// MaterializationSetsDelete fakes DELETE +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. The delete returns a +// long-running operation too, settling with an empty response. +func (s *FakeWorkspace) MaterializationSetsDelete(setID string) Response { + defer s.LockUnlock()() + + existing, ok := s.MaterializationSets[setID] + if !ok { + return materializationSetNotFound(setID) + } + + allIDs := map[string]bool{} + for _, member := range existing.MaterializedFeatures { + allIDs[member.MaterializedFeatureId] = true + } + targets := materializationSetDeletionTargets(existing.MaterializedFeatures) + if failures := materializationSetTeardownFailures(targets, allIDs); len(failures) > 0 { + // The set is kept so the members that still need deleting can be retried. + return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, + partialMaterializationSetChangeErrorCode(failures), + fmt.Sprintf("Deleted %d of %d materialized features in materialization set '%s'. "+ + "The operation's progress lists which deletion succeeded, which failed and why.", + 0, len(failures), setID), + func(metadata *ml.MaterializationSetOperationMetadata) { + metadata.DeletionProgress = &ml.MaterializationSetDeletionProgress{ + FailedMaterializedFeatures: failures, + } + }) + } + + delete(s.MaterializationSets, setID) + + return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, struct{}{}) +} + +// OperationsGet fakes GET /api/2.0/feature-engineering/operations/{operation_id}. +func (s *FakeWorkspace) OperationsGet(operationID string) Response { + defer s.LockUnlock()() + + operation, ok := s.Operations["operations/"+operationID] + if !ok { + // Unlike every other message here, this wording is not taken from the backend: an unknown + // operation is resolved inside the entity-store client rather than from + // FeatureStoreMessages. No case reaches this branch, so nothing pins it either way. + return materializationSetError(http.StatusNotFound, "NOT_FOUND", + fmt.Sprintf("Operation with name 'operations/%s' does not exist.", operationID)) + } + return Response{Body: operation} +} + +// effectiveCronScheduleMode treats an unset mode as MANUAL, so spelling MANUAL out is no change. +func effectiveCronScheduleMode(mode ml.CronScheduleMode) ml.CronScheduleMode { + if mode == "" { + return ml.CronScheduleModeManual + } + return mode +} + +// isCronScheduleChangeRequested mirrors the backend's comparison. The mode is always compared, so +// dropping DERIVED reads as asking to stop deriving; a DERIVED schedule's expression and timezone +// belong to the service, so leaving them out asks for nothing. That is what keeps a sawtooth +// feature updatable, where comparing the field as a whole would reject every update after create. +func isCronScheduleChangeRequested(requested, existing *ml.CronSchedule) bool { + if requested == nil || existing == nil { + return false + } + requestedMode := effectiveCronScheduleMode(requested.Mode) + if requestedMode != effectiveCronScheduleMode(existing.Mode) { + return true + } + if requestedMode == ml.CronScheduleModeDerived { + if requested.CronExpression != "" && requested.CronExpression != existing.CronExpression { + return true + } + } else if requested.CronExpression != existing.CronExpression { + return true + } + return requested.TimezoneId != "" && requested.TimezoneId != existing.TimezoneId +} + +// materializationSetDeletionTargets groups each feature's members into the one a delete is applied +// to and the ones it cascades to. Deleting a feature's offline member deletes its online ones; a +// feature with no offline member is deleted through each online one. +func materializationSetDeletionTargets(members []ml.MaterializedFeature) [][]ml.MaterializedFeature { + byFeature := map[string][]ml.MaterializedFeature{} + var featureOrder []string + for _, member := range members { + if _, seen := byFeature[member.FeatureName]; !seen { + featureOrder = append(featureOrder, member.FeatureName) + } + byFeature[member.FeatureName] = append(byFeature[member.FeatureName], member) + } + + var targets [][]ml.MaterializedFeature + for _, featureName := range featureOrder { + var offline, online []ml.MaterializedFeature + for _, member := range byFeature[featureName] { + if member.IsOnline { + online = append(online, member) + } else { + offline = append(offline, member) + } + } + if len(offline) == 0 { + for _, member := range online { + targets = append(targets, []ml.MaterializedFeature{member}) + } + continue + } + // Assumes one offline destination per feature, as the backend does. + targets = append(targets, append([]ml.MaterializedFeature{offline[0]}, online...)) + } + return targets +} + +// MaterializationSetsUpdate fakes PATCH +// /api/2.0/feature-engineering/materialization-sets/{materialization_set_id}. Field updates are +// validated before any deletion, so the destructive half is never attempted on a set that could +// not be fully updated. +func (s *FakeWorkspace) MaterializationSetsUpdate(req Request, setID string) Response { + defer s.LockUnlock()() + + existing, ok := s.MaterializationSets[setID] + if !ok { + return materializationSetNotFound(setID) + } + + var requested ml.MaterializationSet + if err := json.Unmarshal(req.Body, &requested); err != nil { + return materializationSetInternalError(err) + } + + if len(requested.MaterializedFeatures) == 0 { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Updating materialization set '%s' with no materialized features is not "+ + "supported. Delete the set instead.", setID)) + } + + storedByKey := map[string]ml.MaterializedFeature{} + for _, member := range existing.MaterializedFeatures { + storedByKey[materializationSetMemberKey(member)] = member + } + + // Match each requested member to a stored one. A requested member matching nothing is an + // addition, which this endpoint does not support. + matched := make([]ml.MaterializedFeature, 0, len(requested.MaterializedFeatures)) + requestedKeys := map[string]bool{} + for _, member := range requested.MaterializedFeatures { + stored, found := storedByKey[materializationSetMemberKey(member)] + if !found { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Materialized features cannot be added to an existing materialization "+ + "set, and feature '%s' does not match any member of this set. Recreate the "+ + "set instead.", member.FeatureName)) + } + if isCronScheduleChangeRequested(member.CronScheduleTrigger, stored.CronScheduleTrigger) { + return materializedFeatureFieldCannotBeChanged(member.FeatureName, + stored.MaterializedFeatureId, "trigger") + } + // An unset state asks for no change: the server falls back to the stored one before + // comparing, so a member the config never declares is not read as a change. + if member.PipelineScheduleState != "" && + member.PipelineScheduleState != stored.PipelineScheduleState { + return materializedFeatureFieldCannotBeChanged(member.FeatureName, + stored.MaterializedFeatureId, "pipeline_schedule_state") + } + requestedKeys[materializationSetMemberKey(member)] = true + matched = append(matched, stored) + } + + // Delete the members the request omits. Because a delete cascades from a feature's offline + // member to its online ones, a removal that would also reach a member the request still lists + // cannot be expressed and is refused before anything is written. The server groups the removed + // members by feature and names that feature's removed ids, so the message is built the same way. + removedIDs := map[string]bool{} + featureNameToRemovedIDs := map[string][]string{} + for _, member := range existing.MaterializedFeatures { + if !requestedKeys[materializationSetMemberKey(member)] { + removedIDs[member.MaterializedFeatureId] = true + featureNameToRemovedIDs[member.FeatureName] = append( + featureNameToRemovedIDs[member.FeatureName], member.MaterializedFeatureId) + } + } + for _, target := range materializationSetDeletionTargets(existing.MaterializedFeatures) { + var unrequested []string + requestedForChange := false + for _, member := range target { + if removedIDs[member.MaterializedFeatureId] { + requestedForChange = true + } else { + unrequested = append(unrequested, member.MaterializedFeatureId) + } + } + // A target the caller asked nothing about is not in play. + if !requestedForChange || len(unrequested) == 0 { + continue + } + featureName := target[0].FeatureName + removedForFeature := append([]string(nil), featureNameToRemovedIDs[featureName]...) + sort.Strings(removedForFeature) + // The server has a message for each direction. target[0] is the member a delete is applied + // to, so naming it means the delete reaches past what the request asked for, and omitting + // it means the named members have no delete of their own. + if removedIDs[target[0].MaterializedFeatureId] { + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Deleting materialized features %s of feature '%s' also deletes "+ + "materialized features %s, which the request does not delete. Delete the "+ + "unrequested materialized features as well.", + quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested))) + } + return materializationSetError(http.StatusBadRequest, "INVALID_PARAMETER_VALUE", + fmt.Sprintf("Materialized features %s of feature '%s' are deleted only by deleting "+ + "materialized features %s, which the request does not supply. Remove those as "+ + "well.", + quotedMaterializedFeatureIDs(removedForFeature), featureName, + quotedMaterializedFeatureIDs(unrequested))) + } + + targets := materializationSetDeletionTargets(existing.MaterializedFeatures) + if failures := materializationSetTeardownFailures(targets, removedIDs); len(failures) > 0 { + // Nothing is written: the one target this fake can fail is the only target, so no member + // of the change got through. + return s.materializationSetFailedOperationResponse(setID, existing.MaterializedFeatures, + partialMaterializationSetChangeErrorCode(failures), + fmt.Sprintf("Changed %d of %d materialized features in materialization set '%s'. "+ + "The operation's progress lists which update succeeded, which failed and why.", + 0, len(failures), setID), + func(metadata *ml.MaterializationSetOperationMetadata) { + metadata.UpdateProgress = &ml.MaterializationSetUpdateProgress{ + FailedMaterializedFeatures: failures, + } + }) + } + + existing.MaterializedFeatures = matched + s.MaterializationSets[setID] = existing + + return s.materializationSetOperationResponse(setID, existing.MaterializedFeatures, existing) +}