Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
52 commits
Select commit Hold shift + click to select a range
4288b81
Add explicit deployment state modes and optional DMS history
shreyas-goenka Oct 21, 2026
bbbf210
Document deployment state modes and history error handling
shreyas-goenka Oct 22, 2026
cfa1615
Use a fixed history drain timeout and clarify mode documentation
shreyas-goenka Oct 22, 2026
50ed19e
Scale DMS history drain budget and cap requests at one minute
shreyas-goenka Oct 22, 2026
aa5e06a
Set history drain budget to five seconds per operation
shreyas-goenka Oct 22, 2026
f734f16
Rename best-effort history feature and store computed drain timeout
shreyas-goenka Oct 22, 2026
b9f937e
Use a fixed DMS buffer and wait for queue space
shreyas-goenka Oct 22, 2026
4cd4e08
Clarify the file-mode guard in version completion
shreyas-goenka Oct 22, 2026
34a4070
Use direct enum comparisons for deployment state modes
shreyas-goenka Oct 22, 2026
399b173
Remove redundant version-completion commentary
shreyas-goenka Oct 22, 2026
72fe431
Make state save and delete modes explicit
shreyas-goenka Oct 22, 2026
a215726
Guard version completion with RecordsHistory
shreyas-goenka Oct 22, 2026
3042a15
Restore WithRecovery documentation
shreyas-goenka Oct 22, 2026
c6a030a
Use an unbounded DMS queue and enforce timeouts only at drain
shreyas-goenka Oct 22, 2026
85cb141
Remove redundant mode validation and clarify history errors
shreyas-goenka Oct 22, 2026
7818b7a
Document Open behavior for each deployment state mode
shreyas-goenka Oct 22, 2026
0f7af03
Separate write initialization from mode transitions
shreyas-goenka Oct 22, 2026
58b0549
Restore DMS helper call for serial lookup
shreyas-goenka Oct 22, 2026
09d854c
Use both version counters for file-backed deployment history
shreyas-goenka Oct 22, 2026
09161b1
Clear deployment state during reset and version completion
shreyas-goenka Oct 22, 2026
5beb2cd
Simplify WAL feature recovery to use header features directly
shreyas-goenka Oct 23, 2026
3dc54f1
Explain why WAL feature changes need persistence
shreyas-goenka Oct 23, 2026
535cda7
Explain state source validation when upgrading to write
shreyas-goenka Oct 23, 2026
c8c6be1
Use one writable DMS mode and skip DMS write upgrades
shreyas-goenka Oct 23, 2026
1f230f8
Name WAL setup explicitly and spell out mode branches
shreyas-goenka Oct 23, 2026
cc16973
Preserve the ordinary WAL path and separate history branches
shreyas-goenka Oct 23, 2026
efae162
Remove redundant operation buffer write assertion
shreyas-goenka Oct 23, 2026
05e55a9
Reset state unconditionally when completing a version
shreyas-goenka Oct 23, 2026
76c0343
Inline normal file persistence in save and delete mode cases
shreyas-goenka Oct 23, 2026
198e51d
Clarify which recording fields survive finalization and why
shreyas-goenka Oct 23, 2026
d8206d6
Remove optional best-effort marker check when opening state
shreyas-goenka Oct 23, 2026
33449b7
Use stored modes for write setup and explicit drain policies
shreyas-goenka Oct 23, 2026
e695feb
Preserve DMS header timing for plans and downgrade warnings
shreyas-goenka Oct 23, 2026
8c94c94
Inline DMS mode checks when opening deployment state
shreyas-goenka Oct 23, 2026
a735d8e
Remove optional WAL mode checks and requested failure test
shreyas-goenka Oct 23, 2026
d61e89b
Remove defensive panic when recording after drain
shreyas-goenka Oct 23, 2026
ad5ce69
Simplify unbounded queue coordination with a condition variable
shreyas-goenka Oct 23, 2026
22b54a5
Simplify operation buffering with 100ms idle polling
shreyas-goenka Oct 23, 2026
2333f44
Preserve existing DMS behavior and restore acceptance coverage
shreyas-goenka Oct 23, 2026
9978aab
Explain releasing the empty history queue backing array
shreyas-goenka Oct 23, 2026
90f22a1
Clarify history draining polling and cancellation with short comments
shreyas-goenka Oct 23, 2026
203338b
Clarify that file history does not backpressure the deployment engine
shreyas-goenka Oct 23, 2026
2a4e9cb
Explain the expected cancellation error in the history worker
shreyas-goenka Oct 23, 2026
cc302bc
Document polling as the initial history queue design
shreyas-goenka Oct 23, 2026
3066b3d
Continue recording after upload failures and reduce mode test scope
shreyas-goenka Oct 23, 2026
7afb46e
Remove broad mode tests and retain two small state edge cases
shreyas-goenka Oct 23, 2026
dae6556
Remove the header-only history feature recovery test
shreyas-goenka Oct 23, 2026
c60e9d9
Size history channels and drain budgets from staged operations
shreyas-goenka Oct 23, 2026
ab75f6e
Remove redundant locking detail from record comment
shreyas-goenka Oct 23, 2026
3d75aa8
Name deployment state modes explicitly in Drain documentation
shreyas-goenka Oct 23, 2026
529f38c
Explain the stalled writer drain test setup and assertions
shreyas-goenka Oct 23, 2026
1723931
Explain fake clock behavior in the history drain test
shreyas-goenka Oct 23, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions bundle/direct/bind.go
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
// The state is opened without a DMS client, so the writes below record nothing;
// phases.Bind and phases.Unbind refuse to run at all when recording is enabled.
var checkStateDB dstate.DeploymentState
if err := checkStateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{}); err != nil {
if err := checkStateDB.Open(ctx, statePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{}); err != nil {
// State that cannot be read is not the same as state without a binding:
// the resource may well be managed already, and binding on top of it
// would take over a resource whose ownership was never checked. A state
Expand Down Expand Up @@ -95,7 +95,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
}

// Open temp state
err := b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(false), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{})
err := b.StateDB.Open(ctx, tmpStatePath, dstate.WritingFile, dstate.WithRecovery(false), dstate.OpenDmsArgs{})
if err != nil {
os.Remove(tmpStatePath)
return nil, err
Expand All @@ -118,7 +118,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
log.Infof(ctx, "Bound %s to id=%s (in temp state)", resourceKey, resourceID)

// First plan + update: populate state with resolved config
err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{})
err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})
if err != nil {
os.Remove(tmpStatePath)
return nil, err
Expand Down Expand Up @@ -167,7 +167,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
return nil, fmt.Errorf("compacting state: %w", err)
}

err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{})
err = b.StateDB.Open(ctx, tmpStatePath, dstate.WritingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})
if err != nil {
os.Remove(tmpStatePath)
return nil, err
Expand All @@ -187,7 +187,7 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac
}

// Second plan: this is the plan to present to the user (change between remote resource and config)
err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(false), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{})
err = b.StateDB.Open(ctx, tmpStatePath, dstate.ReadingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})
if err != nil {
os.Remove(tmpStatePath)
return nil, err
Expand Down Expand Up @@ -237,7 +237,7 @@ func (result *BindResult) Cancel() {
// Unbind removes a resource from direct engine state without deleting
// the workspace resource. Also removes associated permissions/grants entries.
func (b *DeploymentBundle) Unbind(ctx context.Context, statePath, resourceKey string) error {
err := b.StateDB.Open(ctx, statePath, dstate.WithRecovery(true), dstate.WithWrite(true), dstate.WithDeploymentHistory(false), dstate.OpenDmsArgs{})
err := b.StateDB.Open(ctx, statePath, dstate.WritingFile, dstate.WithRecovery(true), dstate.OpenDmsArgs{})
if err != nil {
if errors.Is(err, dstate.ErrUnsettingRecording) {
return errors.New("unbind is not supported for a bundle target that records deployment history")
Expand Down
36 changes: 36 additions & 0 deletions bundle/direct/dstate/mode.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package dstate

// Mode selects the source of resource state and the persistence required for writes.
type Mode int

const (
// ReadingFile reads resources.json and optionally recovers a leftover WAL.
// State writes are disabled.
ReadingFile Mode = iota

// WritingFile records state changes in WAL.
// Finalize merges them into resources.json. DMS is unused.
WritingFile

// WritingDMS reads and writes resource state through DMS without creating a WAL.
// DMS errors fail the deployment.
WritingDMS

// WritingFileWithHistory writes state through WAL to resources.json and records DMS history.
// DMS errors and drain timeouts let the deployment continue.
WritingFileWithHistory

// WritingFileWithHistoryEnforced writes state through WAL to resources.json and records DMS history.
// DMS errors and drain timeouts let the deployment continue.
// Persists deployment_history_best_effort to reject clients that cannot honor history recording.
WritingFileWithHistoryEnforced
)

func (m Mode) isWritable() bool {
switch m {
case WritingFile, WritingDMS, WritingFileWithHistory, WritingFileWithHistoryEnforced:
return true
default:
return false
}
}
Loading
Loading