From 27cbee2c2760cf1c856f134a9ef55666d96c0b5d Mon Sep 17 00:00:00 2001 From: davem-bis <68955845+davem-bis@users.noreply.github.com> Date: Thu, 20 Aug 2026 15:54:09 +0100 Subject: [PATCH 1/4] Updated representativeness for indirect_non_breaking models to match deployability. Signed-off-by: davem-bis <68955845+davem-bis@users.noreply.github.com> --- sqlmesh/core/snapshot/definition.py | 3 ++- tests/core/test_snapshot.py | 19 +++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/sqlmesh/core/snapshot/definition.py b/sqlmesh/core/snapshot/definition.py index 0c9635a7c2..7ddddc6b0c 100644 --- a/sqlmesh/core/snapshot/definition.py +++ b/sqlmesh/core/snapshot/definition.py @@ -1720,8 +1720,9 @@ def create( # Similarly, if the model depends on past and the start date is not aligned with the # model's start, we should consider this snapshot non-deployable. this_deployable = False + if not snapshot.is_paused or ( - snapshot.is_indirect_non_breaking and snapshot.intervals + not snapshot.is_indirect_non_breaking and snapshot.intervals ): # This snapshot represents what's currently deployed in prod. representative_shared_version_ids.add(node) diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py index 64bee7f472..7096b29352 100644 --- a/tests/core/test_snapshot.py +++ b/tests/core/test_snapshot.py @@ -2407,32 +2407,44 @@ def test_earliest_start_date(sushi_context: Context): def test_deployability_index(make_snapshot): + # Breaking change - should be both representantive / deployable snapshot_a = make_snapshot(SqlModel(name="a", query=parse_one("SELECT 1"))) snapshot_a.categorize_as(SnapshotChangeCategory.BREAKING) + # Forward only breaking change - cannot be representative / deployable due to forward only snapshot_b = make_snapshot(SqlModel(name="b", query=parse_one("SELECT 1"))) snapshot_b.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) snapshot_b.parents = (snapshot_a.snapshot_id,) + # Indirect breaking - cannot be representative / deployable due to forward only non-deployable parent snapshot_c = make_snapshot(SqlModel(name="c", query=parse_one("SELECT 1"))) snapshot_c.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) snapshot_c.parents = (snapshot_b.snapshot_id,) + # Indirect breaking - cannot be representative / deployable due to forward only non-deployable parent snapshot_d = make_snapshot(SqlModel(name="d", query=parse_one("SELECT 1"))) snapshot_d.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) snapshot_d.parents = (snapshot_b.snapshot_id, snapshot_a.snapshot_id) + # Non breaking - representative / deployable due to no breaking changes snapshot_e = make_snapshot(SqlModel(name="e", query=parse_one("SELECT 1"))) snapshot_e.categorize_as(SnapshotChangeCategory.NON_BREAKING) + # Indirect breaking - can be representative / deployable due to deployable parents snapshot_f = make_snapshot(SqlModel(name="f", query=parse_one("SELECT 1"))) snapshot_f.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) snapshot_f.parents = (snapshot_e.snapshot_id, snapshot_a.snapshot_id) + # Indirect non breaking - cannot be representative / deployable due to possible data drift from prod snapshot_g = make_snapshot(SqlModel(name="g", query=parse_one("SELECT 1"))) snapshot_g.intervals = [(to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))] snapshot_g.categorize_as(SnapshotChangeCategory.INDIRECT_NON_BREAKING) snapshot_g.parents = (snapshot_e.snapshot_id,) + + # Indirect breaking - cannot be representative / deployable due to indirect non breaking non-deployable parent + snapshot_h = make_snapshot(SqlModel(name="h", query=parse_one("SELECT 1"))) + snapshot_h.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) + snapshot_h.parents = (snapshot_g.snapshot_id,) snapshots = { s.snapshot_id: s @@ -2444,6 +2456,7 @@ def test_deployability_index(make_snapshot): snapshot_e, snapshot_f, snapshot_g, + snapshot_h, ] } @@ -2452,18 +2465,20 @@ def test_deployability_index(make_snapshot): assert deployability_index.is_deployable(snapshot_a) assert deployability_index.is_deployable(snapshot_e) assert deployability_index.is_deployable(snapshot_f) - assert not deployability_index.is_deployable(snapshot_g) assert not deployability_index.is_deployable(snapshot_b) assert not deployability_index.is_deployable(snapshot_c) assert not deployability_index.is_deployable(snapshot_d) + assert not deployability_index.is_deployable(snapshot_g) + assert not deployability_index.is_deployable(snapshot_h) assert deployability_index.is_representative(snapshot_a) assert deployability_index.is_representative(snapshot_e) assert deployability_index.is_representative(snapshot_f) - assert deployability_index.is_representative(snapshot_g) assert not deployability_index.is_representative(snapshot_b) assert not deployability_index.is_representative(snapshot_c) assert not deployability_index.is_representative(snapshot_d) + assert not deployability_index.is_representative(snapshot_g) + assert not deployability_index.is_representative(snapshot_h) all_deployable_index = deployability_index.all_deployable() assert all(all_deployable_index.is_deployable(s) for s in snapshots.values()) From 95f9af215f3ce4e63a894bd8a2df07fb19c3c9ef Mon Sep 17 00:00:00 2001 From: davem-bis <68955845+davem-bis@users.noreply.github.com> Date: Thu, 20 Aug 2026 16:29:27 +0100 Subject: [PATCH 2/4] Updated test objects names to be more descriptive. Signed-off-by: davem-bis <68955845+davem-bis@users.noreply.github.com> --- tests/core/test_snapshot.py | 118 +++++++++++++++++++++--------------- 1 file changed, 69 insertions(+), 49 deletions(-) diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py index 7096b29352..9c76707fff 100644 --- a/tests/core/test_snapshot.py +++ b/tests/core/test_snapshot.py @@ -2408,77 +2408,97 @@ def test_earliest_start_date(sushi_context: Context): def test_deployability_index(make_snapshot): # Breaking change - should be both representantive / deployable - snapshot_a = make_snapshot(SqlModel(name="a", query=parse_one("SELECT 1"))) - snapshot_a.categorize_as(SnapshotChangeCategory.BREAKING) + snapshot_breaking = make_snapshot(SqlModel(name="a", query=parse_one("SELECT 1"))) + snapshot_breaking.categorize_as(SnapshotChangeCategory.BREAKING) # Forward only breaking change - cannot be representative / deployable due to forward only - snapshot_b = make_snapshot(SqlModel(name="b", query=parse_one("SELECT 1"))) - snapshot_b.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) - snapshot_b.parents = (snapshot_a.snapshot_id,) + snapshot_breaking_forward_only = make_snapshot(SqlModel(name="b", query=parse_one("SELECT 1"))) + snapshot_breaking_forward_only.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + snapshot_breaking_forward_only.parents = (snapshot_breaking.snapshot_id,) # Indirect breaking - cannot be representative / deployable due to forward only non-deployable parent - snapshot_c = make_snapshot(SqlModel(name="c", query=parse_one("SELECT 1"))) - snapshot_c.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) - snapshot_c.parents = (snapshot_b.snapshot_id,) + snapshot_indirect_breaking_1 = make_snapshot(SqlModel(name="c", query=parse_one("SELECT 1"))) + snapshot_indirect_breaking_1.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) + snapshot_indirect_breaking_1.parents = (snapshot_breaking_forward_only.snapshot_id,) # Indirect breaking - cannot be representative / deployable due to forward only non-deployable parent - snapshot_d = make_snapshot(SqlModel(name="d", query=parse_one("SELECT 1"))) - snapshot_d.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) - snapshot_d.parents = (snapshot_b.snapshot_id, snapshot_a.snapshot_id) + snapshot_indirect_breaking_2 = make_snapshot(SqlModel(name="d", query=parse_one("SELECT 1"))) + snapshot_indirect_breaking_2.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) + snapshot_indirect_breaking_2.parents = ( + snapshot_breaking_forward_only.snapshot_id, + snapshot_breaking.snapshot_id, + ) # Non breaking - representative / deployable due to no breaking changes - snapshot_e = make_snapshot(SqlModel(name="e", query=parse_one("SELECT 1"))) - snapshot_e.categorize_as(SnapshotChangeCategory.NON_BREAKING) + snapshot_non_breaking = make_snapshot(SqlModel(name="e", query=parse_one("SELECT 1"))) + snapshot_non_breaking.categorize_as(SnapshotChangeCategory.NON_BREAKING) # Indirect breaking - can be representative / deployable due to deployable parents - snapshot_f = make_snapshot(SqlModel(name="f", query=parse_one("SELECT 1"))) - snapshot_f.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) - snapshot_f.parents = (snapshot_e.snapshot_id, snapshot_a.snapshot_id) + snapshot_indirect_breaking_deployable_parents = make_snapshot( + SqlModel(name="f", query=parse_one("SELECT 1")) + ) + snapshot_indirect_breaking_deployable_parents.categorize_as( + SnapshotChangeCategory.INDIRECT_BREAKING + ) + snapshot_indirect_breaking_deployable_parents.parents = ( + snapshot_non_breaking.snapshot_id, + snapshot_breaking.snapshot_id, + ) # Indirect non breaking - cannot be representative / deployable due to possible data drift from prod - snapshot_g = make_snapshot(SqlModel(name="g", query=parse_one("SELECT 1"))) - snapshot_g.intervals = [(to_timestamp("2023-01-01"), to_timestamp("2023-01-02"))] - snapshot_g.categorize_as(SnapshotChangeCategory.INDIRECT_NON_BREAKING) - snapshot_g.parents = (snapshot_e.snapshot_id,) - + snapshot_indirect_non_breaking = make_snapshot(SqlModel(name="g", query=parse_one("SELECT 1"))) + snapshot_indirect_non_breaking.intervals = [ + (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) + ] + snapshot_indirect_non_breaking.categorize_as(SnapshotChangeCategory.INDIRECT_NON_BREAKING) + snapshot_indirect_non_breaking.parents = (snapshot_non_breaking.snapshot_id,) + # Indirect breaking - cannot be representative / deployable due to indirect non breaking non-deployable parent - snapshot_h = make_snapshot(SqlModel(name="h", query=parse_one("SELECT 1"))) - snapshot_h.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) - snapshot_h.parents = (snapshot_g.snapshot_id,) + snapshot_indirect_breaking_non_deployable_parents = make_snapshot( + SqlModel(name="h", query=parse_one("SELECT 1")) + ) + snapshot_indirect_breaking_non_deployable_parents.categorize_as( + SnapshotChangeCategory.INDIRECT_BREAKING + ) + snapshot_indirect_breaking_non_deployable_parents.parents = ( + snapshot_indirect_non_breaking.snapshot_id, + ) snapshots = { s.snapshot_id: s for s in [ - snapshot_a, - snapshot_b, - snapshot_c, - snapshot_d, - snapshot_e, - snapshot_f, - snapshot_g, - snapshot_h, + snapshot_breaking, + snapshot_breaking_forward_only, + snapshot_indirect_breaking_1, + snapshot_indirect_breaking_2, + snapshot_non_breaking, + snapshot_indirect_breaking_deployable_parents, + snapshot_indirect_non_breaking, + snapshot_indirect_breaking_non_deployable_parents, ] } deployability_index = DeployabilityIndex.create(snapshots) - assert deployability_index.is_deployable(snapshot_a) - assert deployability_index.is_deployable(snapshot_e) - assert deployability_index.is_deployable(snapshot_f) - assert not deployability_index.is_deployable(snapshot_b) - assert not deployability_index.is_deployable(snapshot_c) - assert not deployability_index.is_deployable(snapshot_d) - assert not deployability_index.is_deployable(snapshot_g) - assert not deployability_index.is_deployable(snapshot_h) - - assert deployability_index.is_representative(snapshot_a) - assert deployability_index.is_representative(snapshot_e) - assert deployability_index.is_representative(snapshot_f) - assert not deployability_index.is_representative(snapshot_b) - assert not deployability_index.is_representative(snapshot_c) - assert not deployability_index.is_representative(snapshot_d) - assert not deployability_index.is_representative(snapshot_g) - assert not deployability_index.is_representative(snapshot_h) + assert deployability_index.is_deployable(snapshot_breaking) + assert deployability_index.is_deployable(snapshot_non_breaking) + assert deployability_index.is_deployable(snapshot_indirect_breaking_deployable_parents) + assert not deployability_index.is_deployable(snapshot_breaking_forward_only) + assert not deployability_index.is_deployable(snapshot_indirect_breaking_1) + assert not deployability_index.is_deployable(snapshot_indirect_breaking_2) + assert not deployability_index.is_deployable(snapshot_indirect_non_breaking) + assert not deployability_index.is_deployable(snapshot_indirect_breaking_non_deployable_parents) + + assert deployability_index.is_representative(snapshot_breaking) + assert deployability_index.is_representative(snapshot_non_breaking) + assert deployability_index.is_representative(snapshot_indirect_breaking_deployable_parents) + assert not deployability_index.is_representative(snapshot_breaking_forward_only) + assert not deployability_index.is_representative(snapshot_indirect_breaking_1) + assert not deployability_index.is_representative(snapshot_indirect_breaking_2) + assert not deployability_index.is_representative(snapshot_indirect_non_breaking) + assert not deployability_index.is_representative( + snapshot_indirect_breaking_non_deployable_parents + ) all_deployable_index = deployability_index.all_deployable() assert all(all_deployable_index.is_deployable(s) for s in snapshots.values()) From b0443a6074da31bd90031fab1ef4e393318362ae Mon Sep 17 00:00:00 2001 From: davem-bis <68955845+davem-bis@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:03:02 +0100 Subject: [PATCH 3/4] Switched missing_interval and physical model logic to use deployability instead of representativeness for determining whether dev or prod instance of each should be used. Signed-off-by: davem-bis <68955845+davem-bis@users.noreply.github.com> --- sqlmesh/core/snapshot/definition.py | 2 +- sqlmesh/core/snapshot/evaluator.py | 2 +- tests/core/test_snapshot.py | 38 ++++++++++++++++++++++++ tests/core/test_snapshot_evaluator.py | 42 +++++++++++++++++++++++++-- 4 files changed, 80 insertions(+), 4 deletions(-) diff --git a/sqlmesh/core/snapshot/definition.py b/sqlmesh/core/snapshot/definition.py index 7ddddc6b0c..9ad4b3320e 100644 --- a/sqlmesh/core/snapshot/definition.py +++ b/sqlmesh/core/snapshot/definition.py @@ -1048,7 +1048,7 @@ def missing_intervals( deployability_index = deployability_index or DeployabilityIndex.all_deployable() intervals = ( - self.intervals if deployability_index.is_representative(self) else self.dev_intervals + self.intervals if deployability_index.is_deployable(self) else self.dev_intervals ) if not self.evaluatable or (self.is_seed and intervals): diff --git a/sqlmesh/core/snapshot/evaluator.py b/sqlmesh/core/snapshot/evaluator.py index 11b3fd1f33..cf4c7de9bc 100644 --- a/sqlmesh/core/snapshot/evaluator.py +++ b/sqlmesh/core/snapshot/evaluator.py @@ -1270,7 +1270,7 @@ def _promote_snapshot( if environment_naming_info.gateway_managed else self.adapter ) - table_name = snapshot.table_name(deployability_index.is_representative(snapshot)) + table_name = snapshot.table_name(deployability_index.is_deployable(snapshot)) view_name = snapshot.qualified_view_name.for_environment( environment_naming_info, dialect=adapter.dialect ) diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py index 9c76707fff..f680276e8b 100644 --- a/tests/core/test_snapshot.py +++ b/tests/core/test_snapshot.py @@ -637,6 +637,43 @@ def test_missing_intervals_start_override_per_model(make_snapshot: t.Callable[.. (to_timestamp("2023-02-07"), to_timestamp("2023-02-08")), ] +def test__missing_intervals__deployable_snapshot_prod_intervals_returned(snapshot: Snapshot): + # Arrange + snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=False) + snapshot.add_interval(start="2020-01-02", end="2020-01-02", is_dev=False) + snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=True) + + snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + deployability_index = DeployabilityIndex.create([snapshot]) + + # Act + missing_intervals = snapshot.missing_intervals(start="2020-01-01", end="2020-01-03", deployability_index=deployability_index) + + # Assert + assert deployability_index.is_deployable(snapshot) + assert missing_intervals == [ + (to_timestamp("2020-01-03"), to_timestamp("2020-01-04")), + ] + +def test__missing_intervals__non_deployable_snapshot_dev_intervals_returned(snapshot: Snapshot): + # Arrange + snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=False) + snapshot.add_interval(start="2020-01-02", end="2020-01-02", is_dev=False) + snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=True) + + snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + deployability_index = DeployabilityIndex.create([snapshot]) + + # Act + missing_intervals = snapshot.missing_intervals(start="2020-01-01", end="2020-01-03", deployability_index=deployability_index) + + # Assert + assert not deployability_index.is_deployable(snapshot) + assert missing_intervals == [ + (to_timestamp("2020-01-02"), to_timestamp("2020-01-03")), # Not missing from prod intervals + (to_timestamp("2020-01-03"), to_timestamp("2020-01-04")), + ] + def test_incremental_time_self_reference(make_snapshot): snapshot = make_snapshot( @@ -3962,3 +3999,4 @@ def test_snapshot_id_and_version_optional_kind_name(): assert snapshot.model_kind_name assert snapshot.is_incremental_unmanaged assert snapshot.full_history_restatement_only + diff --git a/tests/core/test_snapshot_evaluator.py b/tests/core/test_snapshot_evaluator.py index 27bcbe05ae..939fd8bb4b 100644 --- a/tests/core/test_snapshot_evaluator.py +++ b/tests/core/test_snapshot_evaluator.py @@ -307,7 +307,7 @@ def increment_stage_counter(evaluator) -> None: ) -def test_promote(mocker: MockerFixture, adapter_mock, make_snapshot): +def test_promote__deployable__non_dev_physical(mocker: MockerFixture, adapter_mock, make_snapshot): evaluator = SnapshotEvaluator(adapter_mock) model = SqlModel( @@ -319,8 +319,13 @@ def test_promote(mocker: MockerFixture, adapter_mock, make_snapshot): snapshot = make_snapshot(model) snapshot.categorize_as(SnapshotChangeCategory.BREAKING) + deployability_index = DeployabilityIndex.create([snapshot]) - evaluator.promote([snapshot], EnvironmentNamingInfo(name="test_env")) + evaluator.promote( + target_snapshots=[snapshot], + environment_naming_info=EnvironmentNamingInfo(name="test_env"), + deployability_index=deployability_index + ) adapter_mock.transaction.assert_called() adapter_mock.session.assert_called() @@ -336,6 +341,39 @@ def test_promote(mocker: MockerFixture, adapter_mock, make_snapshot): ) +def test_promote__non_deployable__dev_physical(mocker: MockerFixture, adapter_mock, make_snapshot): + evaluator = SnapshotEvaluator(adapter_mock) + + model = SqlModel( + name="test_schema.test_model", + kind=IncrementalByTimeRangeKind(time_column="a"), + storage_format="parquet", + query=parse_one("SELECT a FROM tbl WHERE ds BETWEEN @start_ds and @end_ds"), + ) + + snapshot = make_snapshot(model) + snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) + deployability_index = DeployabilityIndex.create([snapshot]) + + evaluator.promote( + target_snapshots=[snapshot], + environment_naming_info=EnvironmentNamingInfo(name="test_env"), + deployability_index=deployability_index + ) + + adapter_mock.transaction.assert_called() + adapter_mock.session.assert_called() + adapter_mock.create_schema.assert_called_once_with(to_schema("test_schema__test_env")) + adapter_mock.create_view.assert_called_once_with( + "test_schema__test_env.test_model", + parse_one( + f"SELECT * FROM sqlmesh__test_schema.test_schema__test_model__{snapshot.version}__dev" + ), + table_description=None, + column_descriptions=None, + view_properties={}, + ) + def test_demote(mocker: MockerFixture, adapter_mock, make_snapshot): evaluator = SnapshotEvaluator(adapter_mock) From a74d550b4130f308ae6a98c09b409ccd99ab7648 Mon Sep 17 00:00:00 2001 From: davem-bis <68955845+davem-bis@users.noreply.github.com> Date: Wed, 26 Aug 2026 12:57:00 +0100 Subject: [PATCH 4/4] Reverted representativeness logic to before based on testing. Signed-off-by: davem-bis <68955845+davem-bis@users.noreply.github.com> --- sqlmesh/core/snapshot/definition.py | 2 +- tests/core/test_snapshot.py | 126 ++++++++++++++------------ tests/core/test_snapshot_evaluator.py | 7 +- 3 files changed, 72 insertions(+), 63 deletions(-) diff --git a/sqlmesh/core/snapshot/definition.py b/sqlmesh/core/snapshot/definition.py index 9ad4b3320e..1e7acdea4b 100644 --- a/sqlmesh/core/snapshot/definition.py +++ b/sqlmesh/core/snapshot/definition.py @@ -1722,7 +1722,7 @@ def create( this_deployable = False if not snapshot.is_paused or ( - not snapshot.is_indirect_non_breaking and snapshot.intervals + snapshot.is_indirect_non_breaking and snapshot.intervals ): # This snapshot represents what's currently deployed in prod. representative_shared_version_ids.add(node) diff --git a/tests/core/test_snapshot.py b/tests/core/test_snapshot.py index f680276e8b..08e217824e 100644 --- a/tests/core/test_snapshot.py +++ b/tests/core/test_snapshot.py @@ -637,40 +637,46 @@ def test_missing_intervals_start_override_per_model(make_snapshot: t.Callable[.. (to_timestamp("2023-02-07"), to_timestamp("2023-02-08")), ] + def test__missing_intervals__deployable_snapshot_prod_intervals_returned(snapshot: Snapshot): # Arrange snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=False) snapshot.add_interval(start="2020-01-02", end="2020-01-02", is_dev=False) snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=True) - + snapshot.categorize_as(SnapshotChangeCategory.BREAKING) deployability_index = DeployabilityIndex.create([snapshot]) # Act - missing_intervals = snapshot.missing_intervals(start="2020-01-01", end="2020-01-03", deployability_index=deployability_index) + missing_intervals = snapshot.missing_intervals( + start="2020-01-01", end="2020-01-03", deployability_index=deployability_index + ) # Assert assert deployability_index.is_deployable(snapshot) assert missing_intervals == [ (to_timestamp("2020-01-03"), to_timestamp("2020-01-04")), ] - + + def test__missing_intervals__non_deployable_snapshot_dev_intervals_returned(snapshot: Snapshot): # Arrange snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=False) snapshot.add_interval(start="2020-01-02", end="2020-01-02", is_dev=False) snapshot.add_interval(start="2020-01-01", end="2020-01-01", is_dev=True) - + snapshot.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) deployability_index = DeployabilityIndex.create([snapshot]) # Act - missing_intervals = snapshot.missing_intervals(start="2020-01-01", end="2020-01-03", deployability_index=deployability_index) + missing_intervals = snapshot.missing_intervals( + start="2020-01-01", end="2020-01-03", deployability_index=deployability_index + ) # Assert assert not deployability_index.is_deployable(snapshot) assert missing_intervals == [ - (to_timestamp("2020-01-02"), to_timestamp("2020-01-03")), # Not missing from prod intervals + (to_timestamp("2020-01-02"), to_timestamp("2020-01-03")), # Not missing from prod intervals (to_timestamp("2020-01-03"), to_timestamp("2020-01-04")), ] @@ -2444,61 +2450,60 @@ def test_earliest_start_date(sushi_context: Context): def test_deployability_index(make_snapshot): - # Breaking change - should be both representantive / deployable - snapshot_breaking = make_snapshot(SqlModel(name="a", query=parse_one("SELECT 1"))) + # Breaking change - should be both deployable / representative + snapshot_breaking = make_snapshot(SqlModel(name="breaking", query=parse_one("SELECT 1"))) snapshot_breaking.categorize_as(SnapshotChangeCategory.BREAKING) - # Forward only breaking change - cannot be representative / deployable due to forward only - snapshot_breaking_forward_only = make_snapshot(SqlModel(name="b", query=parse_one("SELECT 1"))) + # Forward only breaking change - cannot be deployable / representative due to forward only + snapshot_breaking_forward_only = make_snapshot( + SqlModel(name="forward_only", query=parse_one("SELECT 1")) + ) snapshot_breaking_forward_only.categorize_as(SnapshotChangeCategory.BREAKING, forward_only=True) snapshot_breaking_forward_only.parents = (snapshot_breaking.snapshot_id,) - # Indirect breaking - cannot be representative / deployable due to forward only non-deployable parent - snapshot_indirect_breaking_1 = make_snapshot(SqlModel(name="c", query=parse_one("SELECT 1"))) - snapshot_indirect_breaking_1.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) - snapshot_indirect_breaking_1.parents = (snapshot_breaking_forward_only.snapshot_id,) - - # Indirect breaking - cannot be representative / deployable due to forward only non-deployable parent - snapshot_indirect_breaking_2 = make_snapshot(SqlModel(name="d", query=parse_one("SELECT 1"))) - snapshot_indirect_breaking_2.categorize_as(SnapshotChangeCategory.INDIRECT_BREAKING) - snapshot_indirect_breaking_2.parents = ( + # Indirect breaking - cannot be deployable / representative due to forward only non-representative parent + snapshot_indirect_breaking_non_representative_parent = make_snapshot( + SqlModel(name="indirect_breaking_non_representative_parent", query=parse_one("SELECT 1")) + ) + snapshot_indirect_breaking_non_representative_parent.categorize_as( + SnapshotChangeCategory.INDIRECT_BREAKING + ) + snapshot_indirect_breaking_non_representative_parent.parents = ( snapshot_breaking_forward_only.snapshot_id, - snapshot_breaking.snapshot_id, ) - # Non breaking - representative / deployable due to no breaking changes - snapshot_non_breaking = make_snapshot(SqlModel(name="e", query=parse_one("SELECT 1"))) - snapshot_non_breaking.categorize_as(SnapshotChangeCategory.NON_BREAKING) - - # Indirect breaking - can be representative / deployable due to deployable parents - snapshot_indirect_breaking_deployable_parents = make_snapshot( - SqlModel(name="f", query=parse_one("SELECT 1")) + # Indirect breaking - can be deployable / representative due to forward only representative parent + snapshot_indirect_breaking_representative_parent = make_snapshot( + SqlModel(name="indirect_breaking_representative_parent", query=parse_one("SELECT 1")) ) - snapshot_indirect_breaking_deployable_parents.categorize_as( + snapshot_indirect_breaking_representative_parent.categorize_as( SnapshotChangeCategory.INDIRECT_BREAKING ) - snapshot_indirect_breaking_deployable_parents.parents = ( - snapshot_non_breaking.snapshot_id, - snapshot_breaking.snapshot_id, + snapshot_indirect_breaking_representative_parent.parents = (snapshot_breaking.snapshot_id,) + + # Non breaking - deployable / representative due to no breaking changes + snapshot_non_breaking = make_snapshot( + SqlModel(name="non_breaking", query=parse_one("SELECT 1")) ) + snapshot_non_breaking.categorize_as(SnapshotChangeCategory.NON_BREAKING) - # Indirect non breaking - cannot be representative / deployable due to possible data drift from prod - snapshot_indirect_non_breaking = make_snapshot(SqlModel(name="g", query=parse_one("SELECT 1"))) + # Indirect non breaking - can be representative but not deployable + snapshot_indirect_non_breaking = make_snapshot( + SqlModel(name="indirect_non_breaking", query=parse_one("SELECT 1")) + ) snapshot_indirect_non_breaking.intervals = [ (to_timestamp("2023-01-01"), to_timestamp("2023-01-02")) ] snapshot_indirect_non_breaking.categorize_as(SnapshotChangeCategory.INDIRECT_NON_BREAKING) snapshot_indirect_non_breaking.parents = (snapshot_non_breaking.snapshot_id,) - # Indirect breaking - cannot be representative / deployable due to indirect non breaking non-deployable parent - snapshot_indirect_breaking_non_deployable_parents = make_snapshot( - SqlModel(name="h", query=parse_one("SELECT 1")) + # Breaking with non-representative parent - cannot be deployable due to non-representative parent + snapshot_breaking_non_representative_parent = make_snapshot( + SqlModel(name="breaking_non_deployable_parents", query=parse_one("SELECT 1")) ) - snapshot_indirect_breaking_non_deployable_parents.categorize_as( - SnapshotChangeCategory.INDIRECT_BREAKING - ) - snapshot_indirect_breaking_non_deployable_parents.parents = ( - snapshot_indirect_non_breaking.snapshot_id, + snapshot_breaking_non_representative_parent.categorize_as(SnapshotChangeCategory.BREAKING) + snapshot_breaking_non_representative_parent.parents = ( + snapshot_breaking_forward_only.snapshot_id, ) snapshots = { @@ -2506,37 +2511,41 @@ def test_deployability_index(make_snapshot): for s in [ snapshot_breaking, snapshot_breaking_forward_only, - snapshot_indirect_breaking_1, - snapshot_indirect_breaking_2, + snapshot_indirect_breaking_non_representative_parent, + snapshot_indirect_breaking_representative_parent, snapshot_non_breaking, - snapshot_indirect_breaking_deployable_parents, snapshot_indirect_non_breaking, - snapshot_indirect_breaking_non_deployable_parents, + snapshot_breaking_non_representative_parent, ] } deployability_index = DeployabilityIndex.create(snapshots) assert deployability_index.is_deployable(snapshot_breaking) - assert deployability_index.is_deployable(snapshot_non_breaking) - assert deployability_index.is_deployable(snapshot_indirect_breaking_deployable_parents) + assert deployability_index.is_representative(snapshot_breaking) + assert not deployability_index.is_deployable(snapshot_breaking_forward_only) - assert not deployability_index.is_deployable(snapshot_indirect_breaking_1) - assert not deployability_index.is_deployable(snapshot_indirect_breaking_2) - assert not deployability_index.is_deployable(snapshot_indirect_non_breaking) - assert not deployability_index.is_deployable(snapshot_indirect_breaking_non_deployable_parents) + assert not deployability_index.is_representative(snapshot_breaking_forward_only) - assert deployability_index.is_representative(snapshot_breaking) + assert deployability_index.is_deployable(snapshot_non_breaking) assert deployability_index.is_representative(snapshot_non_breaking) - assert deployability_index.is_representative(snapshot_indirect_breaking_deployable_parents) - assert not deployability_index.is_representative(snapshot_breaking_forward_only) - assert not deployability_index.is_representative(snapshot_indirect_breaking_1) - assert not deployability_index.is_representative(snapshot_indirect_breaking_2) - assert not deployability_index.is_representative(snapshot_indirect_non_breaking) + + assert not deployability_index.is_deployable( + snapshot_indirect_breaking_non_representative_parent + ) assert not deployability_index.is_representative( - snapshot_indirect_breaking_non_deployable_parents + snapshot_indirect_breaking_non_representative_parent ) + assert deployability_index.is_deployable(snapshot_indirect_breaking_representative_parent) + assert deployability_index.is_representative(snapshot_indirect_breaking_representative_parent) + + assert not deployability_index.is_deployable(snapshot_indirect_non_breaking) + assert deployability_index.is_representative(snapshot_indirect_non_breaking) + + assert not deployability_index.is_deployable(snapshot_breaking_non_representative_parent) + assert not deployability_index.is_representative(snapshot_breaking_non_representative_parent) + all_deployable_index = deployability_index.all_deployable() assert all(all_deployable_index.is_deployable(s) for s in snapshots.values()) assert all(all_deployable_index.is_representative(s) for s in snapshots.values()) @@ -3999,4 +4008,3 @@ def test_snapshot_id_and_version_optional_kind_name(): assert snapshot.model_kind_name assert snapshot.is_incremental_unmanaged assert snapshot.full_history_restatement_only - diff --git a/tests/core/test_snapshot_evaluator.py b/tests/core/test_snapshot_evaluator.py index 939fd8bb4b..3b460eeba6 100644 --- a/tests/core/test_snapshot_evaluator.py +++ b/tests/core/test_snapshot_evaluator.py @@ -307,7 +307,7 @@ def increment_stage_counter(evaluator) -> None: ) -def test_promote__deployable__non_dev_physical(mocker: MockerFixture, adapter_mock, make_snapshot): +def test_promote__deployable__prod_physical(mocker: MockerFixture, adapter_mock, make_snapshot): evaluator = SnapshotEvaluator(adapter_mock) model = SqlModel( @@ -324,7 +324,7 @@ def test_promote__deployable__non_dev_physical(mocker: MockerFixture, adapter_mo evaluator.promote( target_snapshots=[snapshot], environment_naming_info=EnvironmentNamingInfo(name="test_env"), - deployability_index=deployability_index + deployability_index=deployability_index, ) adapter_mock.transaction.assert_called() @@ -358,7 +358,7 @@ def test_promote__non_deployable__dev_physical(mocker: MockerFixture, adapter_mo evaluator.promote( target_snapshots=[snapshot], environment_naming_info=EnvironmentNamingInfo(name="test_env"), - deployability_index=deployability_index + deployability_index=deployability_index, ) adapter_mock.transaction.assert_called() @@ -374,6 +374,7 @@ def test_promote__non_deployable__dev_physical(mocker: MockerFixture, adapter_mo view_properties={}, ) + def test_demote(mocker: MockerFixture, adapter_mock, make_snapshot): evaluator = SnapshotEvaluator(adapter_mock)