From 054d6edf7589c7ce57b8ddf37a2e3d4481f55e93 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 29 Jul 2026 14:30:20 +0000 Subject: [PATCH 01/27] add a generic hashing function --- libs/cache/file_cache.go | 7 +-- libs/cache/file_cache_test.go | 5 +- libs/cache/fingerprint.go | 22 --------- libs/cache/fingerprint_test.go | 34 ------------- libs/hash/hash.go | 21 ++++++++ libs/hash/hash_test.go | 88 ++++++++++++++++++++++++++++++++++ 6 files changed, 116 insertions(+), 61 deletions(-) delete mode 100644 libs/cache/fingerprint.go delete mode 100644 libs/cache/fingerprint_test.go create mode 100644 libs/hash/hash.go create mode 100644 libs/hash/hash_test.go diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index 163cbf3bacc..d86b5fcfcb9 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -12,6 +12,7 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/hash" "github.com/databricks/cli/libs/log" ) @@ -155,7 +156,7 @@ func (fc *fileCache) putJSON(ctx context.Context, fingerprint any, data []byte) if !fc.cacheEnabled { return } - cacheKey, err := fingerprintToHash(fingerprint) + cacheKey, err := hash.OfJSON(fingerprint) if err != nil { log.Debugf(ctx, "[Local Cache] failed to generate cache key for put: %v", err) return @@ -169,7 +170,7 @@ func (fc *fileCache) getJSON(ctx context.Context, fingerprint any) ([]byte, bool if !fc.cacheEnabled { return nil, false } - cacheKey, err := fingerprintToHash(fingerprint) + cacheKey, err := hash.OfJSON(fingerprint) if err != nil { log.Debugf(ctx, "[Local Cache] failed to generate cache key: %v", err) return nil, false @@ -191,7 +192,7 @@ func (fc *fileCache) addTelemetryMetric(key string) { // but always computes and never returns cached values. func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, error) { // Convert fingerprint to deterministic hash - this is our cache key - cacheKey, err := fingerprintToHash(fingerprint) + cacheKey, err := hash.OfJSON(fingerprint) if err != nil { // Fail open: if we can't generate cache key, just compute directly log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v", err) diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index b20d03ce2c0..0ce9c571ebc 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -12,6 +12,7 @@ import ( "time" "github.com/databricks/cli/libs/env" + "github.com/databricks/cli/libs/hash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -291,7 +292,7 @@ func TestFileCacheInvalidJSON(t *testing.T) { } // Manually write invalid JSON to the cache file - cacheKey, err := fingerprintToHash(fingerprint) + cacheKey, err := hash.OfJSON(fingerprint) require.NoError(t, err) cachePath := fc.getCachePath(cacheKey) err = os.WriteFile(cachePath, []byte("invalid json {{{"), 0o600) @@ -327,7 +328,7 @@ func TestFileCacheCorruptedData(t *testing.T) { } // Write valid JSON but wrong type (string instead of int) - cacheKey, err := fingerprintToHash(fingerprint) + cacheKey, err := hash.OfJSON(fingerprint) require.NoError(t, err) cachePath := fc.getCachePath(cacheKey) err = os.WriteFile(cachePath, []byte(`"not-an-integer"`), 0o600) diff --git a/libs/cache/fingerprint.go b/libs/cache/fingerprint.go deleted file mode 100644 index 8f866405122..00000000000 --- a/libs/cache/fingerprint.go +++ /dev/null @@ -1,22 +0,0 @@ -package cache - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" -) - -// fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. -func fingerprintToHash(fingerprint any) (string, error) { - // Marshal map - json.Marshal sorts map keys alphabetically - data, err := json.Marshal(fingerprint) - if err != nil { - return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) - } - - // Hash for consistent, reasonably-sized key. - // hash[:] converts the [32]byte array returned by Sum256 to a []byte slice. - hash := sha256.Sum256(data) - return hex.EncodeToString(hash[:]), nil -} diff --git a/libs/cache/fingerprint_test.go b/libs/cache/fingerprint_test.go deleted file mode 100644 index 8a85f7944ce..00000000000 --- a/libs/cache/fingerprint_test.go +++ /dev/null @@ -1,34 +0,0 @@ -package cache - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestFingerprintStability tests that the fingerprintToHash function returns the same hash for the same input. -func TestFingerprintStability(t *testing.T) { - fingerprint1 := struct { - Key string `json:"key"` - }{ - Key: "test-key", - } - - fingerprint2 := struct { - Key string `json:"key"` - }{ - Key: "test-key2", - } - - hash1, err := fingerprintToHash(fingerprint1) - require.NoError(t, err) - require.Equal(t, "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101", hash1) - hash2, err := fingerprintToHash(fingerprint2) - require.NoError(t, err) - hash1ToCompare, err := fingerprintToHash(fingerprint1) - require.NoError(t, err) - - assert.Equal(t, hash1ToCompare, hash1) - assert.NotEqual(t, hash1, hash2) -} diff --git a/libs/hash/hash.go b/libs/hash/hash.go new file mode 100644 index 00000000000..ebee3519d38 --- /dev/null +++ b/libs/hash/hash.go @@ -0,0 +1,21 @@ +// Package hash provides deterministic content hashing helpers. +package hash + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" +) + +// OfJSON returns a deterministic sha256 hex digest of v's JSON encoding. +// json.Marshal sorts map keys, so the digest is stable across runs for equal values. +func OfJSON(v any) (string, error) { + data, err := json.Marshal(v) + if err != nil { + return "", fmt.Errorf("failed to marshal value for hashing: %w", err) + } + + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} diff --git a/libs/hash/hash_test.go b/libs/hash/hash_test.go new file mode 100644 index 00000000000..66f365fb845 --- /dev/null +++ b/libs/hash/hash_test.go @@ -0,0 +1,88 @@ +package hash + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestOfJSONStable tests that the hash of the same value is the same across runs. +func TestOfJSONDeterministic(t *testing.T) { + v := struct { + Key string `json:"key"` + }{ + Key: "test-key", + } + + hash, err := OfJSON(v) + require.NoError(t, err) + + hashAgain, err := OfJSON(v) + require.NoError(t, err) + + assert.Equal(t, hash, hashAgain) +} + +// TestOfJSONReordering tests that the hash of a map is the same regardless of the order of the keys. +func TestOfJSONReordering(t *testing.T) { + v1 := map[string]int{"a": 1, "b": 2} + v2 := map[string]int{"b": 2, "a": 1} + + hash1, err := OfJSON(v1) + require.NoError(t, err) + + hash2, err := OfJSON(v2) + require.NoError(t, err) + + assert.Equal(t, hash1, hash2) +} + +// TestOfJSONDifferentValues tests that the hash of different values is different. +func TestOfJSONDifferentValues(t *testing.T) { + v1 := struct { + Key string `json:"key"` + }{ + Key: "test-key", + } + + v2 := struct { + Key string `json:"key"` + }{ + Key: "test-key2", + } + + hash1, err := OfJSON(v1) + require.NoError(t, err) + + hash2, err := OfJSON(v2) + require.NoError(t, err) + + assert.NotEqual(t, hash1, hash2) +} + +// TestOfJSONKnownValues tests that the hash of a known value is the expected value. +func TestOfJSONKnownValues(t *testing.T) { + v := struct { + Key string `json:"key"` + }{ + Key: "test-key", + } + + expectedHash := "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101" + + hash, err := OfJSON(v) + require.NoError(t, err) + assert.Equal(t, expectedHash, hash) +} + +// TestOfJSONUnsupportedTypes tests that the hash of an unsupported type is an error. +func TestOfJSONUnsupportedTypes(t *testing.T) { + v := func() any { + return nil + } + + hash, err := OfJSON(v) + require.Equal(t, "", hash) + assert.Error(t, err) +} From 9579d61b0c3a6393efede1dd80c38c6891ac3a2f Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 29 Jul 2026 14:30:49 +0000 Subject: [PATCH 02/27] add hashed_in_state to the config --- bundle/direct/dresources/config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 91175594a6a..24398d4256a 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -79,6 +79,11 @@ type ResourceLifecycleConfig struct { // BackendDefaults: fields where the backend may set defaults. // When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped. BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"` + + // HashedInState: field paths persisted to state as a content hash ("sha256:") + // instead of the raw value. This is only valid for large, equality-only fields that + // are never read back from state (e.g. serialized dashboards) + HashedInState []string `yaml:"hashed_in_state,omitempty"` } // Config is the root configuration structure for resource lifecycle behavior. @@ -100,6 +105,7 @@ var empty = ResourceLifecycleConfig{ UpdatableIDFields: nil, NormalizeSlash: nil, BackendDefaults: nil, + HashedInState: nil, } func mustParseConfig(data []byte) func() *Config { From ce11220b4d889a7de444ec9033f19f84dc4f704d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Wed, 29 Jul 2026 14:43:39 +0000 Subject: [PATCH 03/27] add compacting for serialized dashboards --- bundle/direct/dresources/resources.yml | 5 ++ bundle/direct/dresources/state_compaction.go | 87 +++++++++++++++++++ .../dresources/state_compaction_test.go | 86 ++++++++++++++++++ 3 files changed, 178 insertions(+) create mode 100644 bundle/direct/dresources/state_compaction.go create mode 100644 bundle/direct/dresources/state_compaction_test.go diff --git a/bundle/direct/dresources/resources.yml b/bundle/direct/dresources/resources.yml index c9df091551f..344a9cec8d3 100644 --- a/bundle/direct/dresources/resources.yml +++ b/bundle/direct/dresources/resources.yml @@ -462,6 +462,11 @@ resources: - field: dataset_schema reason: input_only + # "serialized_dashboard" holds inlined dashboard JSON that is never read back + # from state, so we persist only its content hash. + hashed_in_state: + - serialized_dashboard + genie_spaces: ignore_remote_changes: # serialized_space locally (structured YAML) and remotely (JSON string) will differ diff --git a/bundle/direct/dresources/state_compaction.go b/bundle/direct/dresources/state_compaction.go new file mode 100644 index 00000000000..27b25177248 --- /dev/null +++ b/bundle/direct/dresources/state_compaction.go @@ -0,0 +1,87 @@ +package dresources + +import ( + "errors" + "fmt" + "reflect" + "strings" + + "github.com/databricks/cli/libs/hash" + "github.com/databricks/cli/libs/structs/structaccess" +) + +// stateHashPrefix marks a state value that holds a content hash instead of the +// raw value. Since this is part of the on-disk state format, changing it is not +// backwards compatible. +const stateHashPrefix = "sha256_hashed_in_state:" + +// hashStateValue returns a content-hash placeholder ("sha256_hashed_in_state:") over the JSON +// encoding of v. It is used to store large, equality-only fields (e.g. a dashboard's +// serialized_dashboard) compactly in state instead of their full contents. +// +// It is idempotent and stable: nil, an empty string, and a value that is already a +// placeholder are returned unchanged, so re-compacting an already-compact state and +// comparing a fresh config against stored state both behave predictably. +func hashStateValue(v any) (any, error) { + if s, ok := v.(string); ok { + if s == "" || strings.HasPrefix(s, stateHashPrefix) { + return v, nil + } + } + + if v == nil { + return v, nil + } + + hashed, err := hash.OfJSON(v) + if err != nil { + return nil, err + } + + return stateHashPrefix + hashed, nil +} + +// CompactState returns a copy of state with every field declared in cfg.HashedInState +// replaced by a content hash, so the state persists only the hash and not the full +// contents. It is applied both before persisting state and to every value entering the +// state diff, so stored and compared values share one form. The caller's value is never +// mutated (it is reused for the deploy API call, which needs the full contents). +// +// Returns state unchanged when no fields are declared or state is not a non-nil pointer. +func CompactState(cfg *ResourceLifecycleConfig, state any) (any, error) { + if cfg == nil || len(cfg.HashedInState) == 0 { + return state, nil + } + + rv := reflect.ValueOf(state) + if rv.Kind() != reflect.Pointer || rv.IsNil() { + return state, nil + } + + // Shallow copy so the caller's value (reused for the deploy) is untouched. This is + // safe only because every hashed_in_state field is a top-level scalar (e.g. + // serialized_dashboard): SetByString overwrites it on the copy directly. A field + // reached through a shared pointer/slice/map would need a deep copy here. + out := reflect.New(rv.Type().Elem()) + out.Elem().Set(rv.Elem()) + compacted := out.Interface() + + for _, field := range cfg.HashedInState { + current, err := structaccess.GetByString(compacted, field) + if err != nil { + if _, ok := errors.AsType[*structaccess.NotFoundError](err); ok { + continue + } + return nil, fmt.Errorf("compacting state field %q: %w", field, err) + } + hashed, err := hashStateValue(current) + if err != nil { + return nil, fmt.Errorf("compacting state field %q: %w", field, err) + } + if err := structaccess.SetByString(compacted, field, hashed); err != nil { + return nil, fmt.Errorf("compacting state field %q: %w", field, err) + } + } + + return compacted, nil +} diff --git a/bundle/direct/dresources/state_compaction_test.go b/bundle/direct/dresources/state_compaction_test.go new file mode 100644 index 00000000000..73e43cc3839 --- /dev/null +++ b/bundle/direct/dresources/state_compaction_test.go @@ -0,0 +1,86 @@ +package dresources + +import ( + "strings" + "testing" + + "github.com/databricks/cli/bundle/config/resources" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestCompactStateNoDeclaredFields verifies CompactState is a no-op for a resource +// type with no hashed_in_state declaration and for a nil config, returning the same +// value untouched. +func TestCompactStateNoDeclaredFields(t *testing.T) { + state := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: `{"a":1}`}} + + out, err := CompactState(GetResourceConfig("jobs"), state) + require.NoError(t, err) + assert.Same(t, state, out.(*DashboardState)) + + out, err = CompactState(nil, state) + require.NoError(t, err) + assert.Same(t, state, out.(*DashboardState)) +} + +// TestCompactStateMigratesLegacyFullContent verifies that a legacy state holding the +// full serialized_dashboard and a config holding identical content compact to the +// same hash, so a diff computed after hashing-on-read shows no spurious change and +// the next save rewrites the state compactly. +func TestCompactStateMigratesLegacyFullContent(t *testing.T) { + content := `{"pages":[{"name":"p1"}]}` + legacy := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: content}} + config := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: content}} + + cfg := GetResourceConfig("dashboards") + compactedLegacy, err := CompactState(cfg, legacy) + require.NoError(t, err) + compactedConfig, err := CompactState(cfg, config) + require.NoError(t, err) + + legacyHash := compactedLegacy.(*DashboardState).SerializedDashboard + assert.Equal(t, compactedConfig.(*DashboardState).SerializedDashboard, legacyHash) + assert.True(t, strings.HasPrefix(legacyHash.(string), stateHashPrefix)) +} + +// TestHashStateValue verifies hashStateValue adds the state hash prefix and produces +// a stable placeholder: the same content always hashes to the same value and +// different content differs. The hash body itself is covered by libs/hash. +func TestHashStateValue(t *testing.T) { + stringHash, err := hashStateValue("hello") + require.NoError(t, err) + require.IsType(t, "", stringHash) + assert.True(t, strings.HasPrefix(stringHash.(string), stateHashPrefix)) + + again, err := hashStateValue("hello") + require.NoError(t, err) + assert.Equal(t, stringHash, again) + + other, err := hashStateValue("world") + require.NoError(t, err) + assert.NotEqual(t, stringHash, other) +} + +// TestHashStateValueIdempotent verifies re-hashing an existing placeholder returns it +// unchanged, so re-compacting an already-compact state does not double-hash. +func TestHashStateValueIdempotent(t *testing.T) { + hashed, err := hashStateValue("some content") + require.NoError(t, err) + + again, err := hashStateValue(hashed) + require.NoError(t, err) + assert.Equal(t, hashed, again) +} + +// TestHashStateValueEmptyAndNil verifies empty and nil values pass through unchanged, +// since there is nothing to hash. +func TestHashStateValueEmptyAndNil(t *testing.T) { + empty, err := hashStateValue("") + require.NoError(t, err) + assert.Empty(t, empty) + + null, err := hashStateValue(nil) + require.NoError(t, err) + assert.Nil(t, null) +} From b3c4e4b335cc353c3bf0315a78a0044dc31efdb6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 08:22:21 +0000 Subject: [PATCH 04/27] add compaction details in README --- bundle/direct/dresources/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index da9e9edab28..1d494783624 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -47,6 +47,16 @@ If the API may return a slice's elements in a different order between calls (e.g The state struct is serialized to JSON and persisted between deploys. Backward incompatible changes will result in a drift, which depending on field behaviour might result in recreate. See dstate/migrate.go on how to handle state migration. +## hashed_in_state: storing large fields as content hashes + +Declare a field under `hashed_in_state` in `resources.yml` when it holds large content that is only ever compared for equality and never read back from state. The engine then persists only a `sha256_hashed_in_state:` content hash for that field (via `CompactState`), and — crucially — hashes it on **every** value entering the diff: saved state, the local config, and the remapped remote. Once the saved value is a hash, all three sides must be hashes or the comparisons (`Old==New` for a local change, `Remote==New` for remote drift) would be hash-vs-content nonsense. The full contents stay only in the plan's `new_state` and are sent to the API on every deploy, so the deploy is unaffected. + +A field qualifies if it is large (a small field gains nothing — the hash placeholder is ~70 bytes) and is never read back from state by any code path (e.g. not consumed raw by `OverrideChangeDesc` or by state export). + +**`hashed_in_state` is orthogonal to `ignore_remote_changes`.** Because the remote side is hashed too, remote drift on a hashed field is detected as `hash != hash` — so a field can be `hashed_in_state` *without* being `ignore_remote_changes` (as long as the server echoes the value back unchanged). The two are declared independently. + +`dashboards.serialized_dashboard` happens to need both, for unrelated reasons: it is `hashed_in_state` because the inlined dashboard JSON is large, and it is *separately* `ignore_remote_changes` because the **server normalizes** it (adds `pageType`, reorders keys) so its remote hash never equals the config hash — drift is detected via `etag` instead. The plan therefore reports the field as a hash on all three sides. No state version bump is needed: legacy full-content state is hashed on read for comparison and rewritten compactly on the next save. + ## RemapState and missing remote fields Do not populate a field in `RemapState` by mapping it from a differently-named field in `RemoteType`. When a field is absent from `RemoteType` the engine automatically suppresses remote drift for it (reason: `missing_in_remote`), which means any value `RemapState` sets there is invisible to drift detection — real remote changes go undetected. From a381f5e54e1100adfefcf3dc6bd06e16afd564b4 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 08:40:56 +0000 Subject: [PATCH 05/27] add compact state to apply, bind, bundle_plan and bundle_state --- bundle/direct/apply.go | 18 +++++++++++++---- bundle/direct/bind.go | 18 ++++++++++++++++- bundle/direct/bundle_plan.go | 37 +++++++++++++++++++++++++++++++++-- bundle/migrate/build_state.go | 11 ++++++++++- 4 files changed, 76 insertions(+), 8 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index b3c46036c53..2473c499a98 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -75,7 +75,7 @@ func (d *DeploymentUnit) Create(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = d.compactAndSaveState(db, newID, newState) if err != nil { return fmt.Errorf("saving state after creating id=%s: %w", newID, err) } @@ -149,7 +149,7 @@ func (d *DeploymentUnit) Update(ctx context.Context, db *dstate.DeploymentState, return err } - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = d.compactAndSaveState(db, id, newState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -193,7 +193,7 @@ func (d *DeploymentUnit) UpdateWithID(ctx context.Context, db *dstate.Deployment return err } - err = db.SaveState(d.ResourceKey, newID, newState, d.DependsOn) + err = d.compactAndSaveState(db, newID, newState) if err != nil { return fmt.Errorf("saving state id=%s: %w", oldID, err) } @@ -276,7 +276,7 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return fmt.Errorf("resizing id=%s: %w", id, err) } - err = db.SaveState(d.ResourceKey, id, newState, d.DependsOn) + err = d.compactAndSaveState(db, id, newState) if err != nil { return fmt.Errorf("saving state id=%s: %w", id, err) } @@ -284,6 +284,16 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return nil } +// compactAndSaveState compacts the state (replacing fields declared hashed_in_state +// with content hashes, see dresources.CompactState) before persisting it +func (d *DeploymentUnit) compactAndSaveState(db *dstate.DeploymentState, id string, newState any) error { + compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), newState) + if err != nil { + return fmt.Errorf("compacting state: %w", err) + } + return db.SaveState(d.ResourceKey, id, compacted, d.DependsOn) +} + func parseState(destType reflect.Type, raw json.RawMessage) (any, error) { destPtr := reflect.New(destType).Interface() err := json.Unmarshal(raw, destPtr) diff --git a/bundle/direct/bind.go b/bundle/direct/bind.go index 9760ce95666..dcd9bf28742 100644 --- a/bundle/direct/bind.go +++ b/bundle/direct/bind.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/bundle/config" "github.com/databricks/cli/bundle/deployplan" + "github.com/databricks/cli/bundle/direct/dresources" "github.com/databricks/cli/bundle/direct/dstate" "github.com/databricks/cli/libs/log" "github.com/databricks/cli/libs/structs/structaccess" @@ -145,13 +146,28 @@ func (b *DeploymentBundle) Bind(ctx context.Context, client *databricks.Workspac } } + // Compact hashed_in_state fields (e.g. a dashboard's serialized_dashboard) so + // the state we persist here matches what a later deploy writes. Otherwise the + // next plan would compare this raw value against the hashed config side and + // report a spurious change on the resource we just bound. + adapter, err := b.getAdapterForKey(resourceKey) + if err != nil { + os.Remove(tmpStatePath) + return nil, err + } + compacted, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value) + if err != nil { + os.Remove(tmpStatePath) + return nil, fmt.Errorf("compacting state: %w", err) + } + err = b.StateDB.Open(ctx, tmpStatePath, dstate.WithRecovery(true), dstate.WithWrite(true)) if err != nil { os.Remove(tmpStatePath) return nil, err } - err = b.StateDB.SaveState(resourceKey, resourceID, sv.Value, dependsOn) + err = b.StateDB.SaveState(resourceKey, resourceID, compacted, dependsOn) if err != nil { os.Remove(tmpStatePath) return nil, err diff --git a/bundle/direct/bundle_plan.go b/bundle/direct/bundle_plan.go index fda8180cfc4..94e112fded6 100644 --- a/bundle/direct/bundle_plan.go +++ b/bundle/direct/bundle_plan.go @@ -225,6 +225,18 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } + // Compact the saved state so hashed_in_state fields are hashes, matching the + // local and remote sides below. The state read from disk may still hold the full + // contents (written by an older CLI, or the field was only just added to + // hashed_in_state); hashing it on read lines the sides up so an unchanged resource + // shows no change. This only affects the in-memory copy used for the diff; the + // on-disk entry keeps its full contents until the resource is next saved. + savedState, err = dresources.CompactState(adapter.ResourceConfig(), savedState) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: compacting saved state: %w", errorPrefix, err)) + return false + } + // Note, currently we're diffing static structs, not dynamic value. // This means for fields that contain references like ${resources.group.foo.id} we do one of the following: // for strings: comparing unresolved string like "${resoures.group.foo.id}" with actual object id. As long as IDs do not have ${...} format we're good. @@ -236,7 +248,14 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks logdiag.LogError(ctx, fmt.Errorf("%s: internal error: no state cache entry found for %q", errorPrefix, resourceKey)) return false } - localDiff, err := structdiff.GetStructDiff(savedState, sv.Value, adapter.KeyedSlices()) + // Compact a copy for comparison only; sv.Value keeps the full contents, which + // the deploy sends to the API. + localState, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: compacting local state: %w", errorPrefix, err)) + return false + } + localDiff, err := structdiff.GetStructDiff(savedState, localState, adapter.KeyedSlices()) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: diffing local state: %w", errorPrefix, err)) return false @@ -269,7 +288,21 @@ func (b *DeploymentBundle) CalculatePlan(ctx context.Context, client *databricks return false } - remoteDiff, err = structdiff.GetStructDiff(remoteStateComparable, sv.Value, adapter.KeyedSlices()) + // Compact the remapped remote on the same fields, so a hashed_in_state field + // is a hash on all three sides of the diff (saved, local, remote). Once the + // saved value is a hash, every comparison must be hash-vs-hash to be meaningful, + // including remote drift. This keeps hashed_in_state orthogonal to + // ignore_remote_changes: remote drift is still detected as hash != hash, so a + // field can be hashed without being ignored. serialized_dashboard is also + // ignore_remote_changes, but for the independent reason that the server + // normalizes it (see resources.yml). + remoteStateComparable, err = dresources.CompactState(adapter.ResourceConfig(), remoteStateComparable) + if err != nil { + logdiag.LogError(ctx, fmt.Errorf("%s: compacting remote state id=%q: %w", errorPrefix, dbentry.ID, err)) + return false + } + + remoteDiff, err = structdiff.GetStructDiff(remoteStateComparable, localState, adapter.KeyedSlices()) if err != nil { logdiag.LogError(ctx, fmt.Errorf("%s: diffing remote state: %w", errorPrefix, err)) return false diff --git a/bundle/migrate/build_state.go b/bundle/migrate/build_state.go index c08cf01c31f..01c3b039d6e 100644 --- a/bundle/migrate/build_state.go +++ b/bundle/migrate/build_state.go @@ -233,7 +233,16 @@ func BuildStateFromTF( } } - if err := stateDB.SaveState(node, id, sv.Value, dependsOn); err != nil { + // Compact hashed_in_state fields (e.g. a dashboard's serialized_dashboard) so the + // migrated state matches what a native deploy writes; otherwise the first plan + // after migrating from Terraform would compare this raw value against the hashed + // config side and report a spurious change. + compacted, err := dresources.CompactState(adapter.ResourceConfig(), sv.Value) + if err != nil { + return warningsSeen, fmt.Errorf("%s: compacting state: %w", node, err) + } + + if err := stateDB.SaveState(node, id, compacted, dependsOn); err != nil { return warningsSeen, fmt.Errorf("%s: SaveState: %w", node, err) } } From 0375f736f27c0dcad205ebe91a2bbce8062b4878 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 08:41:08 +0000 Subject: [PATCH 06/27] add dashboard tests --- bundle/direct/dresources/dashboard_test.go | 52 ++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/bundle/direct/dresources/dashboard_test.go b/bundle/direct/dresources/dashboard_test.go index 177e1236221..49895cea916 100644 --- a/bundle/direct/dresources/dashboard_test.go +++ b/bundle/direct/dresources/dashboard_test.go @@ -2,9 +2,12 @@ package dresources import ( "encoding/json" + "slices" + "strings" "testing" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -23,3 +26,52 @@ func TestDashboardState_JSONSerialization_PublishedField(t *testing.T) { assert.Contains(t, string(data), `"published":true`) } + +func TestDashboardCompactState(t *testing.T) { + state := &DashboardState{ + DashboardConfig: resources.DashboardConfig{ + DisplayName: "test-dashboard", + Etag: "etag-123", + SerializedDashboard: `{"pages":[{"name":"p1"}]}`, + }, + } + + out, err := CompactState(GetResourceConfig("dashboards"), state) + require.NoError(t, err) + compacted := out.(*DashboardState) + + // serialized_dashboard is replaced by a content hash; other fields are preserved. + require.IsType(t, "", compacted.SerializedDashboard) + assert.True(t, strings.HasPrefix(compacted.SerializedDashboard.(string), stateHashPrefix)) + assert.Equal(t, "test-dashboard", compacted.DisplayName) + assert.Equal(t, "etag-123", compacted.Etag) + + // The original state is not mutated. + assert.Equal(t, `{"pages":[{"name":"p1"}]}`, state.SerializedDashboard) + + // Compacting is idempotent. + out2, err := CompactState(GetResourceConfig("dashboards"), compacted) + require.NoError(t, err) + assert.Equal(t, compacted.SerializedDashboard, out2.(*DashboardState).SerializedDashboard) +} + +// TestDashboardSerializedDashboardStateRules documents that serialized_dashboard carries +// two independent declarations: hashed_in_state (persist only its hash, since the blob is +// large) and ignore_remote_changes (the server normalizes the content, so its remote hash +// never matches the config hash — drift is detected via etag instead). The two are +// orthogonal in general; serialized_dashboard just happens to need both. +func TestDashboardSerializedDashboardStateRules(t *testing.T) { + cfg := GetResourceConfig("dashboards") + path := structpath.NewStringKey(nil, "serialized_dashboard") + + ignoresRemote := false + for _, rule := range cfg.IgnoreRemoteChanges { + if path.HasPatternPrefix(rule.Field) { + ignoresRemote = true + break + } + } + + assert.True(t, slices.Contains(cfg.HashedInState, "serialized_dashboard"), "serialized_dashboard must be declared hashed_in_state") + assert.True(t, ignoresRemote, "serialized_dashboard must be ignore_remote_changes (server normalizes the content)") +} From e4076c7903a0bf523fbb77df0f43b8acc73b155d Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 09:53:29 +0000 Subject: [PATCH 07/27] update acceptance tests --- .../dashboard/recreation/out.state_after_bind.direct.json | 2 +- acceptance/bundle/migrate/dashboards/out.new_state.json | 2 +- .../bundle/migrate/dashboards/out.plan_after_migrate.json | 6 +++--- .../resources/dashboards/detect-change/out.plan.direct.json | 6 +++--- .../bundle/resources/dashboards/simple/out.plan.direct.json | 6 +++--- .../dashboards/unpublish-out-of-band/out.plan.direct.json | 6 +++--- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index eb3a71d91ff..e983edf03bb 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -12,7 +12,7 @@ "etag": [ETAG], "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources", "published": true, - "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}", + "serialized_dashboard": "sha256_hashed_in_state:[DASHBOARD_ID][DASHBOARD_ID]", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" } } diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index ed623b6b487..8930713fa16 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -12,7 +12,7 @@ "etag": "[NUMID]", "parent_path": "/Workspace/Users/[USERNAME]", "published": true, - "serialized_dashboard": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n", + "serialized_dashboard": "sha256_hashed_in_state:9dc171d249749a592c717bb13f2d948d2f1fa5530b82ac308a3df0c48d347ef3", "warehouse_id": "123456" } } diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json index cbdc8b8203b..7873382435f 100644 --- a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json @@ -50,9 +50,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n", - "new": "{\"pages\":[{\"name\":\"02724bf2\",\"displayName\":\"Dashboard test bundle-deploy-dashboard\"}]}\n", - "remote": "{\"pages\":[{\"displayName\":\"Dashboard test bundle-deploy-dashboard\",\"name\":\"02724bf2\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n" + "old": "sha256_hashed_in_state:9dc171d249749a592c717bb13f2d948d2f1fa5530b82ac308a3df0c48d347ef3", + "new": "sha256_hashed_in_state:9dc171d249749a592c717bb13f2d948d2f1fa5530b82ac308a3df0c48d347ef3", + "remote": "sha256_hashed_in_state:aae99d4284b3390b1b35974f7bd4c03603cacd6a5a4a838ca563f652257ab6a7" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json index b00e44d4fe8..e0f06060730 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -59,9 +59,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "{\n \"pages\": [\n {\n \"displayName\": \"New Page\",\n \"layout\": [\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 0\n },\n \"widget\": {\n \"name\": \"82eb9107\",\n \"textbox_spec\": \"# I'm a title\"\n }\n },\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 2\n },\n \"widget\": {\n \"name\": \"ffa6de4f\",\n \"textbox_spec\": \"Text\"\n }\n }\n ],\n \"name\": \"fdd21a3c\"\n }\n ]\n}\n", - "new": "{\n \"pages\": [\n {\n \"displayName\": \"New Page\",\n \"layout\": [\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 0\n },\n \"widget\": {\n \"name\": \"82eb9107\",\n \"textbox_spec\": \"# I'm a title\"\n }\n },\n {\n \"position\": {\n \"height\": 2,\n \"width\": 6,\n \"x\": 0,\n \"y\": 2\n },\n \"widget\": {\n \"name\": \"ffa6de4f\",\n \"textbox_spec\": \"Text\"\n }\n }\n ],\n \"name\": \"fdd21a3c\"\n }\n ]\n}\n", - "remote": "{}\n" + "old": "sha256_hashed_in_state:[ALPHANUMID]", + "new": "sha256_hashed_in_state:[ALPHANUMID]", + "remote": "sha256_hashed_in_state:[ALPHANUMID]" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index 5e6ad9eb0a6..b6c82c919f3 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -50,9 +50,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "{ }\n", - "new": "{ }\n", - "remote": "{}\n" + "old": "sha256_hashed_in_state:cf207804e[NUMID]becbb5703765c4978cd7a8466dbdd[NUMID]e571ad09", + "new": "sha256_hashed_in_state:cf207804e[NUMID]becbb5703765c4978cd7a8466dbdd[NUMID]e571ad09", + "remote": "sha256_hashed_in_state:bb511d1e2723214ac1c3710d[NUMID]d57e31b0145d33d427d9628b209be38" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json index 852cc492879..ceeddde3365 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json @@ -66,9 +66,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}", - "new": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}", - "remote": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}" + "old": "sha256_hashed_in_state:a196fded8b5e0ebe5af2c6bed934d90fe055644de7773bbd84c851b577ff4f19", + "new": "sha256_hashed_in_state:a196fded8b5e0ebe5af2c6bed934d90fe055644de7773bbd84c851b577ff4f19", + "remote": "sha256_hashed_in_state:df70601c54be8ac2a66ac4a01dbd5224f4fa20ca9e8d15a615beab794cfdca08" }, "update_time": { "action": "skip", From 2abd40aa877d86b17e868803038f76c2350d5a16 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 10:51:14 +0000 Subject: [PATCH 08/27] create test for checking serialized_dashboard is stored in state as hash but full content is always sent to the API --- .../dashboard-state-sha/dashboard.tmpl | 1 + .../dashboard-state-sha/databricks.yml.tmpl | 11 +++ .../out.create.serialized.json | 3 + .../out.plan_create.direct.json | 19 +++++ .../out.plan_skip.direct.json | 65 ++++++++++++++++ .../out.plan_update.direct.json | 74 +++++++++++++++++++ .../dashboard-state-sha/out.state.direct.txt | 1 + .../out.state_after_update.direct.txt | 1 + .../dashboard-state-sha/out.test.toml | 5 ++ .../out.update.serialized.json | 3 + .../resources/dashboard-state-sha/output.txt | 26 +++++++ .../resources/dashboard-state-sha/script | 56 ++++++++++++++ .../resources/dashboard-state-sha/test.toml | 25 +++++++ 13 files changed, 290 insertions(+) create mode 100644 acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl create mode 100644 acceptance/bundle/resources/dashboard-state-sha/databricks.yml.tmpl create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.test.toml create mode 100644 acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json create mode 100644 acceptance/bundle/resources/dashboard-state-sha/output.txt create mode 100644 acceptance/bundle/resources/dashboard-state-sha/script create mode 100644 acceptance/bundle/resources/dashboard-state-sha/test.toml diff --git a/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl b/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl new file mode 100644 index 00000000000..bf5c695c6d2 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl @@ -0,0 +1 @@ +{"pages":[{"name":"main","displayName":"$DASH_TITLE"}]} diff --git a/acceptance/bundle/resources/dashboard-state-sha/databricks.yml.tmpl b/acceptance/bundle/resources/dashboard-state-sha/databricks.yml.tmpl new file mode 100644 index 00000000000..a41576273c0 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/databricks.yml.tmpl @@ -0,0 +1,11 @@ +bundle: + name: dashboard-state-sha-$UNIQUE_NAME + +resources: + dashboards: + dashboard1: + display_name: $DASHBOARD_DISPLAY_NAME + warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID + embed_credentials: true + file_path: "dashboard.lvdash.json" + parent_path: /Users/$CURRENT_USER_NAME diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json b/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json new file mode 100644 index 00000000000..c21b142583a --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json @@ -0,0 +1,3 @@ +[ + "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n" +] diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json new file mode 100644 index 00000000000..b9d477135b6 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json @@ -0,0 +1,19 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "plan": { + "resources.dashboards.dashboard1": { + "action": "create", + "new_state": { + "value": { + "display_name": "dashboard-state-sha [UUID]", + "embed_credentials": true, + "parent_path": "/Workspace/Users/[USERNAME]", + "published": true, + "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json new file mode 100644 index 00000000000..208c72cebe4 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json @@ -0,0 +1,65 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.dashboards.dashboard1": { + "action": "skip", + "remote_state": { + "create_time": "[TIMESTAMP]", + "dashboard_id": "[DASHBOARD_ID]", + "display_name": "dashboard-state-sha [UUID]", + "embed_credentials": true, + "etag": [ETAG], + "lifecycle_state": "ACTIVE", + "parent_path": "/Workspace/Users/[USERNAME]", + "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json", + "published": true, + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", + "update_time": "[TIMESTAMP]", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[TIMESTAMP]" + }, + "dashboard_id": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[DASHBOARD_ID]" + }, + "etag": { + "action": "skip", + "reason": "custom", + "old": [ETAG], + "remote": [ETAG] + }, + "lifecycle_state": { + "action": "skip", + "reason": "spec:output_only", + "remote": "ACTIVE" + }, + "path": { + "action": "skip", + "reason": "spec:output_only", + "remote": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json" + }, + "serialized_dashboard": { + "action": "skip", + "reason": "etag_based", + "old": "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a", + "new": "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a", + "remote": "sha256_hashed_in_state:c14d4500f6b61da69067cbce0ac4e0b212f04c155dedad5d04b97bf461903ef5" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[TIMESTAMP]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json new file mode 100644 index 00000000000..923ed737577 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json @@ -0,0 +1,74 @@ +{ + "plan_version": 2, + "cli_version": "[CLI_VERSION]", + "lineage": "[UUID]", + "serial": 1, + "plan": { + "resources.dashboards.dashboard1": { + "action": "update", + "new_state": { + "value": { + "display_name": "dashboard-state-sha [UUID]", + "embed_credentials": true, + "parent_path": "/Workspace/Users/[USERNAME]", + "published": true, + "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + } + }, + "remote_state": { + "create_time": "[TIMESTAMP]", + "dashboard_id": "[DASHBOARD_ID]", + "display_name": "dashboard-state-sha [UUID]", + "embed_credentials": true, + "etag": [ETAG], + "lifecycle_state": "ACTIVE", + "parent_path": "/Workspace/Users/[USERNAME]", + "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json", + "published": true, + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", + "update_time": "[TIMESTAMP]", + "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" + }, + "changes": { + "create_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[TIMESTAMP]" + }, + "dashboard_id": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[DASHBOARD_ID]" + }, + "etag": { + "action": "skip", + "reason": "custom", + "old": [ETAG], + "remote": [ETAG] + }, + "lifecycle_state": { + "action": "skip", + "reason": "spec:output_only", + "remote": "ACTIVE" + }, + "path": { + "action": "skip", + "reason": "spec:output_only", + "remote": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json" + }, + "serialized_dashboard": { + "action": "update", + "old": "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a", + "new": "sha256_hashed_in_state:37ad7625d69f4a6b4e119dc2f0a2ad415c752c22a58ddcc333f[NUMID]a5825", + "remote": "sha256_hashed_in_state:c14d4500f6b61da69067cbce0ac4e0b212f04c155dedad5d04b97bf461903ef5" + }, + "update_time": { + "action": "skip", + "reason": "spec:output_only", + "remote": "[TIMESTAMP]" + } + } + } + } +} diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt b/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt new file mode 100644 index 00000000000..cc4ce8135ce --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt @@ -0,0 +1 @@ +json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a"; diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt b/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt new file mode 100644 index 00000000000..fd85e06379e --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt @@ -0,0 +1 @@ +json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:37ad7625d69f4a6b4e119dc2f0a2ad415c752c22a58ddcc333f[NUMID]a5825"; diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.test.toml b/acceptance/bundle/resources/dashboard-state-sha/out.test.toml new file mode 100644 index 00000000000..2a4f200653a --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.test.toml @@ -0,0 +1,5 @@ +Local = true +Cloud = true +RequiresWarehouse = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json b/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json new file mode 100644 index 00000000000..5c8339e26b5 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json @@ -0,0 +1,3 @@ +[ + "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n" +] diff --git a/acceptance/bundle/resources/dashboard-state-sha/output.txt b/acceptance/bundle/resources/dashboard-state-sha/output.txt new file mode 100644 index 00000000000..bd79441ffe3 --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/output.txt @@ -0,0 +1,26 @@ + +=== Create: plan keeps full content in new_state but reports the diff as a hash +>>> [CLI] bundle plan -o json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +=== Re-plan with no local change is a no-op (server normalization is ignored, etag_based) +>>> [CLI] bundle plan -o json + +=== Edit serialized_dashboard: the hash changes, so the local diff detects an update +>>> [CLI] bundle plan -o json +Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default/files... +Deploying resources... +Updating deployment state... +Deployment complete! + +>>> [CLI] bundle destroy --auto-approve +The following resources will be deleted: + delete resources.dashboards.dashboard1 + +All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/dashboard-state-sha-[UNIQUE_NAME]/default + +Deleting files... +Destroy complete! diff --git a/acceptance/bundle/resources/dashboard-state-sha/script b/acceptance/bundle/resources/dashboard-state-sha/script new file mode 100644 index 00000000000..39fb090585e --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/script @@ -0,0 +1,56 @@ +DASHBOARD_DISPLAY_NAME="dashboard-state-sha $(uuid)" +if [ -z "$CLOUD_ENV" ]; then + export TEST_DEFAULT_WAREHOUSE_ID="warehouse-1234" + echo "warehouse-1234:TEST_DEFAULT_WAREHOUSE_ID" >> ACC_REPLS +fi +export DASHBOARD_DISPLAY_NAME +envsubst < databricks.yml.tmpl > databricks.yml + +render_dashboard() { + DASH_TITLE="$1" envsubst < dashboard.tmpl > dashboard.lvdash.json +} + +cleanup() { + trace $CLI bundle destroy --auto-approve + rm -f out.requests.txt +} +trap cleanup EXIT +rm -f out.requests.txt + +title "Create: plan keeps full content in new_state but reports the diff as a hash" +render_dashboard "Sales Overview" +trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json + +# Deploy. With READPLAN=1 this applies the SAVED plan file instead of re-planning. The plan +# and the persisted state keep only the hash, but the create API call must still send the +# FULL serialized_dashboard. out.create.serialized.json is identical for READPLAN="" and +# READPLAN=1, which proves the saved plan applies the real content rather than the hash. +# Not traced: the deploy command line differs by READPLAN (--plan vs none). +$CLI bundle deploy $(readplanarg out.plan_create.direct.json) + +DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') +echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS + +jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.create.serialized.json +rm -f out.requests.txt + +# Persisted state holds ONLY the content hash, never the dashboard JSON. +print_state.py | gron.py | grep serialized_dashboard > out.state.$DATABRICKS_BUNDLE_ENGINE.txt + +title "Re-plan with no local change is a no-op (server normalization is ignored, etag_based)" +trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json + +title "Edit serialized_dashboard: the hash changes, so the local diff detects an update" +render_dashboard "Sales Overview v2" +trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json + +# Apply the update (in-memory or from the saved plan). The update API call must carry the +# FULL new content; out.update.serialized.json is identical across READPLAN, proving the +# saved plan applies the real content for updates too. +$CLI bundle deploy $(readplanarg out.plan_update.direct.json) + +jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.update.serialized.json +rm -f out.requests.txt + +# State holds the NEW content hash after the update. +print_state.py | gron.py | grep serialized_dashboard > out.state_after_update.$DATABRICKS_BUNDLE_ENGINE.txt diff --git a/acceptance/bundle/resources/dashboard-state-sha/test.toml b/acceptance/bundle/resources/dashboard-state-sha/test.toml new file mode 100644 index 00000000000..ecdc1a0a77a --- /dev/null +++ b/acceptance/bundle/resources/dashboard-state-sha/test.toml @@ -0,0 +1,25 @@ +# Direct-engine hashed-state behaviour for dashboards. Kept outside dashboards/ so it can run +# direct-only: terraform does not hash state, and the saved-plan path (deploy --plan, i.e. +# READPLAN=1) is direct-only. RecordRequests is inherited from resources/test.toml. +Local = true +Cloud = true +RequiresWarehouse = true + +# dashboard.lvdash.json is rendered from dashboard.tmpl in the script (re-rendered for the edit). +Ignore = ["dashboard.lvdash.json"] + +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.READPLAN = ["", "1"] + +[Env] +# Prevent MSYS2 from rewriting /Users/... paths on Windows. +MSYS_NO_PATHCONV = "1" + +# Etag can be both negative and positive. +[[Repls]] +Old = "\"[-0-9]{8,}\"" +New = "[ETAG]" + +[[Repls]] +Old = "\"[0-9]{8,}\"" +New = "[ETAG]" From 1dcc6e49924edd94a29471b2a81b644f9f33564c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 12:02:07 +0000 Subject: [PATCH 09/27] add hash masking rule --- .../recreation/out.state_after_bind.direct.json | 2 +- .../dashboard-state-sha/out.plan_skip.direct.json | 6 +++--- .../dashboard-state-sha/out.plan_update.direct.json | 6 +++--- .../resources/dashboard-state-sha/out.state.direct.txt | 2 +- .../out.state_after_update.direct.txt | 2 +- .../dashboards/detect-change/out.plan.direct.json | 6 +++--- .../resources/dashboards/simple/out.plan.direct.json | 6 +++--- .../unpublish-out-of-band/out.plan.direct.json | 6 +++--- acceptance/bundle/test.toml | 10 ++++++++++ 9 files changed, 28 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index e983edf03bb..c52412ab5fb 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -12,7 +12,7 @@ "etag": [ETAG], "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources", "published": true, - "serialized_dashboard": "sha256_hashed_in_state:[DASHBOARD_ID][DASHBOARD_ID]", + "serialized_dashboard": "sha256_hashed_in_state:[HASH]", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" } } diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json index 208c72cebe4..595950529ab 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json @@ -50,9 +50,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a", - "new": "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a", - "remote": "sha256_hashed_in_state:c14d4500f6b61da69067cbce0ac4e0b212f04c155dedad5d04b97bf461903ef5" + "old": "sha256_hashed_in_state:[HASH][0]", + "new": "sha256_hashed_in_state:[HASH][0]", + "remote": "sha256_hashed_in_state:[HASH][1]" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json index 923ed737577..f6b9a4efa37 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json @@ -59,9 +59,9 @@ }, "serialized_dashboard": { "action": "update", - "old": "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a", - "new": "sha256_hashed_in_state:37ad7625d69f4a6b4e119dc2f0a2ad415c752c22a58ddcc333f[NUMID]a5825", - "remote": "sha256_hashed_in_state:c14d4500f6b61da69067cbce0ac4e0b212f04c155dedad5d04b97bf461903ef5" + "old": "sha256_hashed_in_state:[HASH][0]", + "new": "sha256_hashed_in_state:[HASH][1]", + "remote": "sha256_hashed_in_state:[HASH][2]" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt b/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt index cc4ce8135ce..f2a1bea9c55 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt +++ b/acceptance/bundle/resources/dashboard-state-sha/out.state.direct.txt @@ -1 +1 @@ -json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:1a7a0b46514ad1d8a1b53e2c8bee5d37b39d5c40157b27f93058bf866d2bc41a"; +json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:[HASH]"; diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt b/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt index fd85e06379e..f2a1bea9c55 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt +++ b/acceptance/bundle/resources/dashboard-state-sha/out.state_after_update.direct.txt @@ -1 +1 @@ -json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:37ad7625d69f4a6b4e119dc2f0a2ad415c752c22a58ddcc333f[NUMID]a5825"; +json.state.resources.dashboards.dashboard1.state.serialized_dashboard = "sha256_hashed_in_state:[HASH]"; diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json index e0f06060730..bff656a3a32 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -59,9 +59,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:[ALPHANUMID]", - "new": "sha256_hashed_in_state:[ALPHANUMID]", - "remote": "sha256_hashed_in_state:[ALPHANUMID]" + "old": "sha256_hashed_in_state:[HASH][0]", + "new": "sha256_hashed_in_state:[HASH][0]", + "remote": "sha256_hashed_in_state:[HASH][1]" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index b6c82c919f3..32fe8c1b420 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -50,9 +50,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:cf207804e[NUMID]becbb5703765c4978cd7a8466dbdd[NUMID]e571ad09", - "new": "sha256_hashed_in_state:cf207804e[NUMID]becbb5703765c4978cd7a8466dbdd[NUMID]e571ad09", - "remote": "sha256_hashed_in_state:bb511d1e2723214ac1c3710d[NUMID]d57e31b0145d33d427d9628b209be38" + "old": "sha256_hashed_in_state:[HASH][0]", + "new": "sha256_hashed_in_state:[HASH][0]", + "remote": "sha256_hashed_in_state:[HASH][1]" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json index ceeddde3365..0197870475c 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json @@ -66,9 +66,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:a196fded8b5e0ebe5af2c6bed934d90fe055644de7773bbd84c851b577ff4f19", - "new": "sha256_hashed_in_state:a196fded8b5e0ebe5af2c6bed934d90fe055644de7773bbd84c851b577ff4f19", - "remote": "sha256_hashed_in_state:df70601c54be8ac2a66ac4a01dbd5224f4fa20ca9e8d15a615beab794cfdca08" + "old": "sha256_hashed_in_state:[HASH][0]", + "new": "sha256_hashed_in_state:[HASH][0]", + "remote": "sha256_hashed_in_state:[HASH][1]" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index fca8259a3b5..34279fb94d9 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -22,3 +22,13 @@ New = 'os/[OS]' [[Repls]] Old = ' cicd/github' New = '' + +# Mask the serialized_dashboard content hash before the generic numeric/id rules can +# nibble digit-runs out of the hex (Order below their default). Distinct numbers the +# hashes, so old==new stays one token and a differing remote gets another +# (e.g. [HASH][0] vs [HASH][1]). +[[Repls]] +Old = "sha256_hashed_in_state:[0-9a-f]{64}" +New = "sha256_hashed_in_state:[HASH]" +Order = -100 +Distinct = true From 99f0fcff53d73121e448dea7cb2840ac1f2d82e6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 12:21:11 +0000 Subject: [PATCH 10/27] change order to -1 --- acceptance/bundle/test.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/acceptance/bundle/test.toml b/acceptance/bundle/test.toml index 34279fb94d9..56862f10cca 100644 --- a/acceptance/bundle/test.toml +++ b/acceptance/bundle/test.toml @@ -30,5 +30,5 @@ New = '' [[Repls]] Old = "sha256_hashed_in_state:[0-9a-f]{64}" New = "sha256_hashed_in_state:[HASH]" -Order = -100 +Order = -1 Distinct = true From b2d0c3ceb8ddde901649b940d2a9d80f87d43b3b Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 12:43:04 +0000 Subject: [PATCH 11/27] change comparision from equal to empty --- libs/hash/hash_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/hash/hash_test.go b/libs/hash/hash_test.go index 66f365fb845..7ddb04aa0d5 100644 --- a/libs/hash/hash_test.go +++ b/libs/hash/hash_test.go @@ -83,6 +83,6 @@ func TestOfJSONUnsupportedTypes(t *testing.T) { } hash, err := OfJSON(v) - require.Equal(t, "", hash) + require.Empty(t, hash) assert.Error(t, err) } From 697071d8861cf5c08ec8aeacc06f39c301b32ebe Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 13:03:36 +0000 Subject: [PATCH 12/27] update golden files missed in last update --- acceptance/bundle/migrate/dashboards/out.new_state.json | 2 +- .../bundle/migrate/dashboards/out.plan_after_migrate.json | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/migrate/dashboards/out.new_state.json b/acceptance/bundle/migrate/dashboards/out.new_state.json index 8930713fa16..33d4c0791b5 100644 --- a/acceptance/bundle/migrate/dashboards/out.new_state.json +++ b/acceptance/bundle/migrate/dashboards/out.new_state.json @@ -12,7 +12,7 @@ "etag": "[NUMID]", "parent_path": "/Workspace/Users/[USERNAME]", "published": true, - "serialized_dashboard": "sha256_hashed_in_state:9dc171d249749a592c717bb13f2d948d2f1fa5530b82ac308a3df0c48d347ef3", + "serialized_dashboard": "sha256_hashed_in_state:[HASH]", "warehouse_id": "123456" } } diff --git a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json index 7873382435f..8fb86c2484f 100644 --- a/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json +++ b/acceptance/bundle/migrate/dashboards/out.plan_after_migrate.json @@ -50,9 +50,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:9dc171d249749a592c717bb13f2d948d2f1fa5530b82ac308a3df0c48d347ef3", - "new": "sha256_hashed_in_state:9dc171d249749a592c717bb13f2d948d2f1fa5530b82ac308a3df0c48d347ef3", - "remote": "sha256_hashed_in_state:aae99d4284b3390b1b35974f7bd4c03603cacd6a5a4a838ca563f652257ab6a7" + "old": "sha256_hashed_in_state:[HASH][0]", + "new": "sha256_hashed_in_state:[HASH][0]", + "remote": "sha256_hashed_in_state:[HASH][1]" }, "update_time": { "action": "skip", From 33e7d2d499b7d94270ec41f161fd796c040a9457 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 13:45:11 +0000 Subject: [PATCH 13/27] adjust for windows new lines --- .../bundle/resources/dashboard-state-sha/script | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/acceptance/bundle/resources/dashboard-state-sha/script b/acceptance/bundle/resources/dashboard-state-sha/script index 39fb090585e..bcb03911019 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/script +++ b/acceptance/bundle/resources/dashboard-state-sha/script @@ -19,7 +19,9 @@ rm -f out.requests.txt title "Create: plan keeps full content in new_state but reports the diff as a hash" render_dashboard "Sales Overview" -trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +# Normalize \r\n to \n in the JSON payload to handle Windows line endings (dashboard.tmpl +# is rendered at test time, so its trailing newline is CRLF on Windows). +trace $CLI bundle plan -o json | sed 's/\\r\\n/\\n/g' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json # Deploy. With READPLAN=1 this applies the SAVED plan file instead of re-planning. The plan # and the persisted state keep only the hash, but the create API call must still send the @@ -31,25 +33,25 @@ $CLI bundle deploy $(readplanarg out.plan_create.direct.json) DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS -jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.create.serialized.json +jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt | sed 's/\\r\\n/\\n/g' > out.create.serialized.json rm -f out.requests.txt # Persisted state holds ONLY the content hash, never the dashboard JSON. print_state.py | gron.py | grep serialized_dashboard > out.state.$DATABRICKS_BUNDLE_ENGINE.txt title "Re-plan with no local change is a no-op (server normalization is ignored, etag_based)" -trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | sed 's/\\r\\n/\\n/g' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json title "Edit serialized_dashboard: the hash changes, so the local diff detects an update" render_dashboard "Sales Overview v2" -trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json | sed 's/\\r\\n/\\n/g' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json # Apply the update (in-memory or from the saved plan). The update API call must carry the # FULL new content; out.update.serialized.json is identical across READPLAN, proving the # saved plan applies the real content for updates too. $CLI bundle deploy $(readplanarg out.plan_update.direct.json) -jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.update.serialized.json +jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt | sed 's/\\r\\n/\\n/g' > out.update.serialized.json rm -f out.requests.txt # State holds the NEW content hash after the update. From 8b82654fdb1a447880c37f4950309575881ba130 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 14:15:31 +0000 Subject: [PATCH 14/27] fix windows path rewrite issue --- acceptance/bundle/resources/dashboard-state-sha/test.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/acceptance/bundle/resources/dashboard-state-sha/test.toml b/acceptance/bundle/resources/dashboard-state-sha/test.toml index ecdc1a0a77a..ffe262f960d 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/test.toml +++ b/acceptance/bundle/resources/dashboard-state-sha/test.toml @@ -11,10 +11,6 @@ Ignore = ["dashboard.lvdash.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] -[Env] -# Prevent MSYS2 from rewriting /Users/... paths on Windows. -MSYS_NO_PATHCONV = "1" - # Etag can be both negative and positive. [[Repls]] Old = "\"[-0-9]{8,}\"" From 2bd204a8375e565fac4034dab05a098e03814060 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 15:08:56 +0000 Subject: [PATCH 15/27] update for windows test --- .../resources/dashboard-state-sha/script | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/acceptance/bundle/resources/dashboard-state-sha/script b/acceptance/bundle/resources/dashboard-state-sha/script index bcb03911019..70b49b04399 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/script +++ b/acceptance/bundle/resources/dashboard-state-sha/script @@ -7,7 +7,11 @@ export DASHBOARD_DISPLAY_NAME envsubst < databricks.yml.tmpl > databricks.yml render_dashboard() { - DASH_TITLE="$1" envsubst < dashboard.tmpl > dashboard.lvdash.json + # Strip CR so the rendered content is LF on every platform. dashboard.tmpl is not pinned to + # eol=lf, so on Windows it renders with a CRLF trailing newline; since serialized_dashboard is + # hashed into state, a CRLF here would hash differently than the LF content in a saved plan and + # make the no-op re-plan (READPLAN=1) spuriously report an update. + DASH_TITLE="$1" envsubst < dashboard.tmpl | tr -d '\r' > dashboard.lvdash.json } cleanup() { @@ -19,9 +23,7 @@ rm -f out.requests.txt title "Create: plan keeps full content in new_state but reports the diff as a hash" render_dashboard "Sales Overview" -# Normalize \r\n to \n in the JSON payload to handle Windows line endings (dashboard.tmpl -# is rendered at test time, so its trailing newline is CRLF on Windows). -trace $CLI bundle plan -o json | sed 's/\\r\\n/\\n/g' > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json > out.plan_create.$DATABRICKS_BUNDLE_ENGINE.json # Deploy. With READPLAN=1 this applies the SAVED plan file instead of re-planning. The plan # and the persisted state keep only the hash, but the create API call must still send the @@ -33,25 +35,25 @@ $CLI bundle deploy $(readplanarg out.plan_create.direct.json) DASHBOARD_ID=$($CLI bundle summary --output json | jq -r '.resources.dashboards.dashboard1.id') echo "$DASHBOARD_ID:DASHBOARD_ID" >> ACC_REPLS -jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt | sed 's/\\r\\n/\\n/g' > out.create.serialized.json +jq -s '[.[] | select(.method == "POST" and (.path | endswith("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.create.serialized.json rm -f out.requests.txt # Persisted state holds ONLY the content hash, never the dashboard JSON. print_state.py | gron.py | grep serialized_dashboard > out.state.$DATABRICKS_BUNDLE_ENGINE.txt title "Re-plan with no local change is a no-op (server normalization is ignored, etag_based)" -trace $CLI bundle plan -o json | sed 's/\\r\\n/\\n/g' > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json > out.plan_skip.$DATABRICKS_BUNDLE_ENGINE.json title "Edit serialized_dashboard: the hash changes, so the local diff detects an update" render_dashboard "Sales Overview v2" -trace $CLI bundle plan -o json | sed 's/\\r\\n/\\n/g' > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json +trace $CLI bundle plan -o json > out.plan_update.$DATABRICKS_BUNDLE_ENGINE.json # Apply the update (in-memory or from the saved plan). The update API call must carry the # FULL new content; out.update.serialized.json is identical across READPLAN, proving the # saved plan applies the real content for updates too. $CLI bundle deploy $(readplanarg out.plan_update.direct.json) -jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt | sed 's/\\r\\n/\\n/g' > out.update.serialized.json +jq -s '[.[] | select(.method == "PATCH" and (.path | contains("/api/2.0/lakeview/dashboards")) and .body.serialized_dashboard != null) | .body.serialized_dashboard]' out.requests.txt > out.update.serialized.json rm -f out.requests.txt # State holds the NEW content hash after the update. From 1a78e119374b58ba31936efd9e9258b2504006c3 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 15:18:31 +0000 Subject: [PATCH 16/27] update comment --- bundle/direct/dresources/config.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/config.go b/bundle/direct/dresources/config.go index 24398d4256a..4b1133fb699 100644 --- a/bundle/direct/dresources/config.go +++ b/bundle/direct/dresources/config.go @@ -80,9 +80,10 @@ type ResourceLifecycleConfig struct { // When old and new are nil but remote is set, and the remote value matches allowed values (if specified), the change is skipped. BackendDefaults []BackendDefaultRule `yaml:"backend_defaults,omitempty"` - // HashedInState: field paths persisted to state as a content hash ("sha256:") - // instead of the raw value. This is only valid for large, equality-only fields that - // are never read back from state (e.g. serialized dashboards) + // HashedInState: field paths persisted to state as a content hash + // ("sha256_hashed_in_state:") instead of the raw value. This is only valid + // for large, equality-only fields that are never read back from state + // (e.g. serialized dashboards). HashedInState []string `yaml:"hashed_in_state,omitempty"` } From 44aaae789b03d772bcaee745f9118975170a31a0 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 15:19:35 +0000 Subject: [PATCH 17/27] fix comment nit --- bundle/direct/apply.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index 2473c499a98..45949b0b7e4 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -284,8 +284,8 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, return nil } -// compactAndSaveState compacts the state (replacing fields declared hashed_in_state -// with content hashes, see dresources.CompactState) before persisting it +// compactAndSaveState compacts the state (replacing fields declared in hashed_in_state +// with content hashes, see dresources.CompactState) before persisting it. func (d *DeploymentUnit) compactAndSaveState(db *dstate.DeploymentState, id string, newState any) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), newState) if err != nil { From 602294048c7974181a2a12fc59f47f7954c4ccc9 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 30 Jul 2026 15:34:45 +0000 Subject: [PATCH 18/27] force shallow copy assumption in code --- bundle/direct/dresources/state_compaction.go | 14 +++++++++++ .../dresources/state_compaction_test.go | 25 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/bundle/direct/dresources/state_compaction.go b/bundle/direct/dresources/state_compaction.go index 27b25177248..d91df4be0f6 100644 --- a/bundle/direct/dresources/state_compaction.go +++ b/bundle/direct/dresources/state_compaction.go @@ -8,6 +8,7 @@ import ( "github.com/databricks/cli/libs/hash" "github.com/databricks/cli/libs/structs/structaccess" + "github.com/databricks/cli/libs/structs/structpath" ) // stateHashPrefix marks a state value that holds a content hash instead of the @@ -67,6 +68,19 @@ func CompactState(cfg *ResourceLifecycleConfig, state any) (any, error) { compacted := out.Interface() for _, field := range cfg.HashedInState { + // The shallow copy above only isolates top-level fields: SetByString on a + // depth-1 path reassigns the field on the copy, but a nested path (e.g. + // "foo.bar") is reached through a pointer/slice/map still shared with the + // caller's value, so hashing it would mutate the value reused for the deploy + // API call. Reject nested paths loudly instead of corrupting state silently. + path, err := structpath.ParsePath(field) + if err != nil { + return nil, fmt.Errorf("compacting state field %q: %w", field, err) + } + if path.Len() != 1 { + return nil, fmt.Errorf("hashed_in_state field %q must be a top-level field", field) + } + current, err := structaccess.GetByString(compacted, field) if err != nil { if _, ok := errors.AsType[*structaccess.NotFoundError](err); ok { diff --git a/bundle/direct/dresources/state_compaction_test.go b/bundle/direct/dresources/state_compaction_test.go index 73e43cc3839..ce3be353967 100644 --- a/bundle/direct/dresources/state_compaction_test.go +++ b/bundle/direct/dresources/state_compaction_test.go @@ -5,10 +5,35 @@ import ( "testing" "github.com/databricks/cli/bundle/config/resources" + "github.com/databricks/cli/libs/structs/structpath" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) +// TestHashedInStateFieldsAreTopLevel guards the shallow-copy assumption in CompactState: +// every hashed_in_state path declared in resources.yml must be a top-level field. A nested +// path would be mutated through memory shared with the deploy value (see CompactState), so +// this fails CI the moment such a declaration is added instead of corrupting state at runtime. +func TestHashedInStateFieldsAreTopLevel(t *testing.T) { + for name, rc := range MustLoadConfig().Resources { + for _, field := range rc.HashedInState { + path, err := structpath.ParsePath(field) + require.NoError(t, err, "%s: hashed_in_state field %q", name, field) + assert.Equal(t, 1, path.Len(), "%s: hashed_in_state field %q must be a top-level field", name, field) + } + } +} + +// TestCompactStateRejectsNestedField verifies CompactState errors on a nested +// hashed_in_state path rather than mutating memory shared with the deploy value. +func TestCompactStateRejectsNestedField(t *testing.T) { + cfg := &ResourceLifecycleConfig{HashedInState: []string{"foo.bar"}} + state := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: `{"a":1}`}} + + _, err := CompactState(cfg, state) + require.ErrorContains(t, err, "must be a top-level field") +} + // TestCompactStateNoDeclaredFields verifies CompactState is a no-op for a resource // type with no hashed_in_state declaration and for a nil config, returning the same // value untouched. From e96f19f5108d30df4b0608b2eb6f94a7dbe188ee Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 31 Jul 2026 07:29:24 +0000 Subject: [PATCH 19/27] update test to not run on cloud --- .../bundle/resources/dashboard-state-sha/out.test.toml | 3 +-- acceptance/bundle/resources/dashboard-state-sha/test.toml | 7 +++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.test.toml b/acceptance/bundle/resources/dashboard-state-sha/out.test.toml index 2a4f200653a..71970b719d4 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.test.toml +++ b/acceptance/bundle/resources/dashboard-state-sha/out.test.toml @@ -1,5 +1,4 @@ Local = true -Cloud = true -RequiresWarehouse = true +Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboard-state-sha/test.toml b/acceptance/bundle/resources/dashboard-state-sha/test.toml index ffe262f960d..3b31f0cae74 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/test.toml +++ b/acceptance/bundle/resources/dashboard-state-sha/test.toml @@ -1,9 +1,12 @@ # Direct-engine hashed-state behaviour for dashboards. Kept outside dashboards/ so it can run # direct-only: terraform does not hash state, and the saved-plan path (deploy --plan, i.e. # READPLAN=1) is direct-only. RecordRequests is inherited from resources/test.toml. +# Local-only: this exercises direct-engine state format (serialized_dashboard stored as a +# content hash), which the in-memory testserver covers fully and deterministically. On real +# cloud the read-after-deploy plan/state reads hit dashboard eventual consistency, and unlike +# the tests under dashboards/ this dir doesn't inherit their retry/INJECT_STALE machinery. Local = true -Cloud = true -RequiresWarehouse = true +Cloud = false # dashboard.lvdash.json is rendered from dashboard.tmpl in the script (re-rendered for the edit). Ignore = ["dashboard.lvdash.json"] From 634ec638225634f175406e8605243ffc9621d12c Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 31 Jul 2026 09:35:56 +0000 Subject: [PATCH 20/27] retrigger CI Empty commit to re-run the pipeline; the previous integration run failed only on gcp-linux TestFsCp* tests, which this PR does not touch (unrelated infra flake). Co-authored-by: Isaac From 823ae7dfc4c3a925e941662b7c676111dbbdb652 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 12:56:46 +0000 Subject: [PATCH 21/27] remove hash from libs and inline usage --- bundle/direct/dresources/state_compaction.go | 12 ++- .../dresources/state_compaction_test.go | 2 +- libs/cache/file_cache.go | 7 +- libs/cache/file_cache_test.go | 5 +- libs/cache/fingerprint.go | 22 +++++ libs/cache/fingerprint_test.go | 54 ++++++++++++ libs/hash/hash.go | 21 ----- libs/hash/hash_test.go | 88 ------------------- 8 files changed, 90 insertions(+), 121 deletions(-) create mode 100644 libs/cache/fingerprint.go create mode 100644 libs/cache/fingerprint_test.go delete mode 100644 libs/hash/hash.go delete mode 100644 libs/hash/hash_test.go diff --git a/bundle/direct/dresources/state_compaction.go b/bundle/direct/dresources/state_compaction.go index d91df4be0f6..650fc0faf1f 100644 --- a/bundle/direct/dresources/state_compaction.go +++ b/bundle/direct/dresources/state_compaction.go @@ -1,12 +1,14 @@ package dresources import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" "errors" "fmt" "reflect" "strings" - "github.com/databricks/cli/libs/hash" "github.com/databricks/cli/libs/structs/structaccess" "github.com/databricks/cli/libs/structs/structpath" ) @@ -34,12 +36,14 @@ func hashStateValue(v any) (any, error) { return v, nil } - hashed, err := hash.OfJSON(v) + // json.Marshal sorts map keys, so the digest is stable across runs for equal values. + data, err := json.Marshal(v) if err != nil { - return nil, err + return nil, fmt.Errorf("marshalling value for hashing: %w", err) } + sum := sha256.Sum256(data) - return stateHashPrefix + hashed, nil + return stateHashPrefix + hex.EncodeToString(sum[:]), nil } // CompactState returns a copy of state with every field declared in cfg.HashedInState diff --git a/bundle/direct/dresources/state_compaction_test.go b/bundle/direct/dresources/state_compaction_test.go index ce3be353967..24e4af61ebc 100644 --- a/bundle/direct/dresources/state_compaction_test.go +++ b/bundle/direct/dresources/state_compaction_test.go @@ -71,7 +71,7 @@ func TestCompactStateMigratesLegacyFullContent(t *testing.T) { // TestHashStateValue verifies hashStateValue adds the state hash prefix and produces // a stable placeholder: the same content always hashes to the same value and -// different content differs. The hash body itself is covered by libs/hash. +// different content differs. func TestHashStateValue(t *testing.T) { stringHash, err := hashStateValue("hello") require.NoError(t, err) diff --git a/libs/cache/file_cache.go b/libs/cache/file_cache.go index d86b5fcfcb9..163cbf3bacc 100644 --- a/libs/cache/file_cache.go +++ b/libs/cache/file_cache.go @@ -12,7 +12,6 @@ import ( "github.com/databricks/cli/internal/build" "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/hash" "github.com/databricks/cli/libs/log" ) @@ -156,7 +155,7 @@ func (fc *fileCache) putJSON(ctx context.Context, fingerprint any, data []byte) if !fc.cacheEnabled { return } - cacheKey, err := hash.OfJSON(fingerprint) + cacheKey, err := fingerprintToHash(fingerprint) if err != nil { log.Debugf(ctx, "[Local Cache] failed to generate cache key for put: %v", err) return @@ -170,7 +169,7 @@ func (fc *fileCache) getJSON(ctx context.Context, fingerprint any) ([]byte, bool if !fc.cacheEnabled { return nil, false } - cacheKey, err := hash.OfJSON(fingerprint) + cacheKey, err := fingerprintToHash(fingerprint) if err != nil { log.Debugf(ctx, "[Local Cache] failed to generate cache key: %v", err) return nil, false @@ -192,7 +191,7 @@ func (fc *fileCache) addTelemetryMetric(key string) { // but always computes and never returns cached values. func (fc *fileCache) getOrComputeJSON(ctx context.Context, fingerprint any, compute func(ctx context.Context) ([]byte, error)) ([]byte, error) { // Convert fingerprint to deterministic hash - this is our cache key - cacheKey, err := hash.OfJSON(fingerprint) + cacheKey, err := fingerprintToHash(fingerprint) if err != nil { // Fail open: if we can't generate cache key, just compute directly log.Debugf(ctx, "[Local Cache] failed to generate cache key, computing without cache: %v", err) diff --git a/libs/cache/file_cache_test.go b/libs/cache/file_cache_test.go index 0ce9c571ebc..b20d03ce2c0 100644 --- a/libs/cache/file_cache_test.go +++ b/libs/cache/file_cache_test.go @@ -12,7 +12,6 @@ import ( "time" "github.com/databricks/cli/libs/env" - "github.com/databricks/cli/libs/hash" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -292,7 +291,7 @@ func TestFileCacheInvalidJSON(t *testing.T) { } // Manually write invalid JSON to the cache file - cacheKey, err := hash.OfJSON(fingerprint) + cacheKey, err := fingerprintToHash(fingerprint) require.NoError(t, err) cachePath := fc.getCachePath(cacheKey) err = os.WriteFile(cachePath, []byte("invalid json {{{"), 0o600) @@ -328,7 +327,7 @@ func TestFileCacheCorruptedData(t *testing.T) { } // Write valid JSON but wrong type (string instead of int) - cacheKey, err := hash.OfJSON(fingerprint) + cacheKey, err := fingerprintToHash(fingerprint) require.NoError(t, err) cachePath := fc.getCachePath(cacheKey) err = os.WriteFile(cachePath, []byte(`"not-an-integer"`), 0o600) diff --git a/libs/cache/fingerprint.go b/libs/cache/fingerprint.go new file mode 100644 index 00000000000..8f866405122 --- /dev/null +++ b/libs/cache/fingerprint.go @@ -0,0 +1,22 @@ +package cache + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" +) + +// fingerprintToHash converts any struct to a deterministic string representation for use as a cache key. +func fingerprintToHash(fingerprint any) (string, error) { + // Marshal map - json.Marshal sorts map keys alphabetically + data, err := json.Marshal(fingerprint) + if err != nil { + return "", fmt.Errorf("failed to marshal normalized fingerprint: %w", err) + } + + // Hash for consistent, reasonably-sized key. + // hash[:] converts the [32]byte array returned by Sum256 to a []byte slice. + hash := sha256.Sum256(data) + return hex.EncodeToString(hash[:]), nil +} diff --git a/libs/cache/fingerprint_test.go b/libs/cache/fingerprint_test.go new file mode 100644 index 00000000000..6fb5df37fff --- /dev/null +++ b/libs/cache/fingerprint_test.go @@ -0,0 +1,54 @@ +package cache + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// TestFingerprintStability tests that the fingerprintToHash function returns the same hash for the same input. +func TestFingerprintStability(t *testing.T) { + fingerprint1 := struct { + Key string `json:"key"` + }{ + Key: "test-key", + } + + fingerprint2 := struct { + Key string `json:"key"` + }{ + Key: "test-key2", + } + + hash1, err := fingerprintToHash(fingerprint1) + require.NoError(t, err) + require.Equal(t, "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101", hash1) + hash2, err := fingerprintToHash(fingerprint2) + require.NoError(t, err) + hash1ToCompare, err := fingerprintToHash(fingerprint1) + require.NoError(t, err) + + assert.Equal(t, hash1ToCompare, hash1) + assert.NotEqual(t, hash1, hash2) +} + +// TestFingerprintMapKeyOrder tests that maps hash independently of key insertion order, +// which is what makes a map usable as a cache key (json.Marshal sorts map keys). +func TestFingerprintMapKeyOrder(t *testing.T) { + hash1, err := fingerprintToHash(map[string]int{"a": 1, "b": 2}) + require.NoError(t, err) + + hash2, err := fingerprintToHash(map[string]int{"b": 2, "a": 1}) + require.NoError(t, err) + + assert.Equal(t, hash1, hash2) +} + +// TestFingerprintUnmarshallable tests that a value json.Marshal rejects is reported as an +// error rather than silently hashing to a constant. +func TestFingerprintUnmarshallable(t *testing.T) { + hash, err := fingerprintToHash(func() {}) + assert.Error(t, err) + assert.Empty(t, hash) +} diff --git a/libs/hash/hash.go b/libs/hash/hash.go deleted file mode 100644 index ebee3519d38..00000000000 --- a/libs/hash/hash.go +++ /dev/null @@ -1,21 +0,0 @@ -// Package hash provides deterministic content hashing helpers. -package hash - -import ( - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" -) - -// OfJSON returns a deterministic sha256 hex digest of v's JSON encoding. -// json.Marshal sorts map keys, so the digest is stable across runs for equal values. -func OfJSON(v any) (string, error) { - data, err := json.Marshal(v) - if err != nil { - return "", fmt.Errorf("failed to marshal value for hashing: %w", err) - } - - sum := sha256.Sum256(data) - return hex.EncodeToString(sum[:]), nil -} diff --git a/libs/hash/hash_test.go b/libs/hash/hash_test.go deleted file mode 100644 index 7ddb04aa0d5..00000000000 --- a/libs/hash/hash_test.go +++ /dev/null @@ -1,88 +0,0 @@ -package hash - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -// TestOfJSONStable tests that the hash of the same value is the same across runs. -func TestOfJSONDeterministic(t *testing.T) { - v := struct { - Key string `json:"key"` - }{ - Key: "test-key", - } - - hash, err := OfJSON(v) - require.NoError(t, err) - - hashAgain, err := OfJSON(v) - require.NoError(t, err) - - assert.Equal(t, hash, hashAgain) -} - -// TestOfJSONReordering tests that the hash of a map is the same regardless of the order of the keys. -func TestOfJSONReordering(t *testing.T) { - v1 := map[string]int{"a": 1, "b": 2} - v2 := map[string]int{"b": 2, "a": 1} - - hash1, err := OfJSON(v1) - require.NoError(t, err) - - hash2, err := OfJSON(v2) - require.NoError(t, err) - - assert.Equal(t, hash1, hash2) -} - -// TestOfJSONDifferentValues tests that the hash of different values is different. -func TestOfJSONDifferentValues(t *testing.T) { - v1 := struct { - Key string `json:"key"` - }{ - Key: "test-key", - } - - v2 := struct { - Key string `json:"key"` - }{ - Key: "test-key2", - } - - hash1, err := OfJSON(v1) - require.NoError(t, err) - - hash2, err := OfJSON(v2) - require.NoError(t, err) - - assert.NotEqual(t, hash1, hash2) -} - -// TestOfJSONKnownValues tests that the hash of a known value is the expected value. -func TestOfJSONKnownValues(t *testing.T) { - v := struct { - Key string `json:"key"` - }{ - Key: "test-key", - } - - expectedHash := "1b329dc07a9fa87da7480f6b10cc917a40a4f460ac82aea3d09df477764f3101" - - hash, err := OfJSON(v) - require.NoError(t, err) - assert.Equal(t, expectedHash, hash) -} - -// TestOfJSONUnsupportedTypes tests that the hash of an unsupported type is an error. -func TestOfJSONUnsupportedTypes(t *testing.T) { - v := func() any { - return nil - } - - hash, err := OfJSON(v) - require.Empty(t, hash) - assert.Error(t, err) -} From 1b1fbaa69921d202e46e9ba2ae95489cbfffecb6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 14:36:01 +0000 Subject: [PATCH 22/27] compact only if hashing saves space --- .../dashboard-state-sha/dashboard.tmpl | 2 +- .../resources/dashboard-state-sha/test.toml | 3 + bundle/direct/apply.go | 3 +- bundle/direct/dresources/README.md | 4 +- bundle/direct/dresources/dashboard_test.go | 6 +- bundle/direct/dresources/state_compaction.go | 25 +++- .../dresources/state_compaction_test.go | 120 +++++++++++++++++- 7 files changed, 145 insertions(+), 18 deletions(-) diff --git a/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl b/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl index bf5c695c6d2..d0b0216ccf1 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl +++ b/acceptance/bundle/resources/dashboard-state-sha/dashboard.tmpl @@ -1 +1 @@ -{"pages":[{"name":"main","displayName":"$DASH_TITLE"}]} +{"pages":[{"name":"main","displayName":"$DASH_TITLE","layout":[{"widget":{"name":"counter1"}}]}]} diff --git a/acceptance/bundle/resources/dashboard-state-sha/test.toml b/acceptance/bundle/resources/dashboard-state-sha/test.toml index 3b31f0cae74..6b239a11106 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/test.toml +++ b/acceptance/bundle/resources/dashboard-state-sha/test.toml @@ -9,6 +9,9 @@ Local = true Cloud = false # dashboard.lvdash.json is rendered from dashboard.tmpl in the script (re-rendered for the edit). +# dashboard.tmpl must stay large enough that hashing shrinks the state: content that encodes to +# no more than a placeholder (87 bytes) is persisted raw, which would silently stop this test from +# exercising hashing at all. Ignore = ["dashboard.lvdash.json"] EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] diff --git a/bundle/direct/apply.go b/bundle/direct/apply.go index cf8f62debf6..dea45cdfa41 100644 --- a/bundle/direct/apply.go +++ b/bundle/direct/apply.go @@ -300,7 +300,8 @@ func (d *DeploymentUnit) Resize(ctx context.Context, db *dstate.DeploymentState, } // compactAndSaveState compacts the state (replacing fields declared in hashed_in_state -// with content hashes, see dresources.CompactState) before persisting it. +// with content hashes, see dresources.CompactState) before persisting it. Fields already +// smaller than a hash placeholder are persisted as is. func (d *DeploymentUnit) compactAndSaveState(db *dstate.DeploymentState, id string, newState any) error { compacted, err := dresources.CompactState(d.Adapter.ResourceConfig(), newState) if err != nil { diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index 1f058e85c14..c0b54413f3b 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -57,7 +57,9 @@ on field behaviour might result in recreate. See dstate/migrate.go on how to han Declare a field under `hashed_in_state` in `resources.yml` when it holds large content that is only ever compared for equality and never read back from state. The engine then persists only a `sha256_hashed_in_state:` content hash for that field (via `CompactState`), and — crucially — hashes it on **every** value entering the diff: saved state, the local config, and the remapped remote. Once the saved value is a hash, all three sides must be hashes or the comparisons (`Old==New` for a local change, `Remote==New` for remote drift) would be hash-vs-content nonsense. The full contents stay only in the plan's `new_state` and are sent to the API on every deploy, so the deploy is unaffected. -A field qualifies if it is large (a small field gains nothing — the hash placeholder is ~70 bytes) and is never read back from state by any code path (e.g. not consumed raw by `OverrideChangeDesc` or by state export). +A field qualifies if it is large and is never read back from state by any code path (e.g. not consumed raw by `OverrideChangeDesc` or by state export). + +Hashing is skipped when it would not pay for itself: a value whose JSON encoding is no longer than the 87-byte placeholder is persisted as is. The verdict depends only on the value, so the state being saved and all three sides of the diff agree on it and stay comparable — including when the same field is small for one resource and large for another, or grows past the threshold between deploys. **`hashed_in_state` is orthogonal to `ignore_remote_changes`.** Because the remote side is hashed too, remote drift on a hashed field is detected as `hash != hash` — so a field can be `hashed_in_state` *without* being `ignore_remote_changes` (as long as the server echoes the value back unchanged). The two are declared independently. diff --git a/bundle/direct/dresources/dashboard_test.go b/bundle/direct/dresources/dashboard_test.go index 49895cea916..d7b8baf4f8b 100644 --- a/bundle/direct/dresources/dashboard_test.go +++ b/bundle/direct/dresources/dashboard_test.go @@ -28,11 +28,13 @@ func TestDashboardState_JSONSerialization_PublishedField(t *testing.T) { } func TestDashboardCompactState(t *testing.T) { + requireLargeEnoughToHash(t, largeDashboard) + state := &DashboardState{ DashboardConfig: resources.DashboardConfig{ DisplayName: "test-dashboard", Etag: "etag-123", - SerializedDashboard: `{"pages":[{"name":"p1"}]}`, + SerializedDashboard: largeDashboard, }, } @@ -47,7 +49,7 @@ func TestDashboardCompactState(t *testing.T) { assert.Equal(t, "etag-123", compacted.Etag) // The original state is not mutated. - assert.Equal(t, `{"pages":[{"name":"p1"}]}`, state.SerializedDashboard) + assert.Equal(t, largeDashboard, state.SerializedDashboard) // Compacting is idempotent. out2, err := CompactState(GetResourceConfig("dashboards"), compacted) diff --git a/bundle/direct/dresources/state_compaction.go b/bundle/direct/dresources/state_compaction.go index 650fc0faf1f..6e230d55fdb 100644 --- a/bundle/direct/dresources/state_compaction.go +++ b/bundle/direct/dresources/state_compaction.go @@ -18,13 +18,18 @@ import ( // backwards compatible. const stateHashPrefix = "sha256_hashed_in_state:" +// stateHashPlaceholderLen is the exact size of a placeholder: the prefix followed by a +// SHA-256 digest in hex (sha256.Size bytes, two characters each). +const stateHashPlaceholderLen = len(stateHashPrefix) + sha256.Size*2 + // hashStateValue returns a content-hash placeholder ("sha256_hashed_in_state:") over the JSON // encoding of v. It is used to store large, equality-only fields (e.g. a dashboard's // serialized_dashboard) compactly in state instead of their full contents. // -// It is idempotent and stable: nil, an empty string, and a value that is already a -// placeholder are returned unchanged, so re-compacting an already-compact state and -// comparing a fresh config against stored state both behave predictably. +// It is idempotent and stable: nil, an empty string, a value that is already a +// placeholder, and a value too small to be worth hashing are returned unchanged, so +// re-compacting an already-compact state and comparing a fresh config against stored +// state both behave predictably. func hashStateValue(v any) (any, error) { if s, ok := v.(string); ok { if s == "" || strings.HasPrefix(s, stateHashPrefix) { @@ -41,6 +46,12 @@ func hashStateValue(v any) (any, error) { if err != nil { return nil, fmt.Errorf("marshalling value for hashing: %w", err) } + + // Compare against the JSON encoding, since that is what the state file stores + if len(data) <= stateHashPlaceholderLen { + return v, nil + } + sum := sha256.Sum256(data) return stateHashPrefix + hex.EncodeToString(sum[:]), nil @@ -48,9 +59,11 @@ func hashStateValue(v any) (any, error) { // CompactState returns a copy of state with every field declared in cfg.HashedInState // replaced by a content hash, so the state persists only the hash and not the full -// contents. It is applied both before persisting state and to every value entering the -// state diff, so stored and compared values share one form. The caller's value is never -// mutated (it is reused for the deploy API call, which needs the full contents). +// contents. A field whose contents are already no larger than a placeholder is left as +// is (see hashStateValue). It is applied both before persisting state and to every value +// entering the state diff, so stored and compared values share one form. The caller's +// value is never mutated (it is reused for the deploy API call, which needs the full +// contents). // // Returns state unchanged when no fields are declared or state is not a non-nil pointer. func CompactState(cfg *ResourceLifecycleConfig, state any) (any, error) { diff --git a/bundle/direct/dresources/state_compaction_test.go b/bundle/direct/dresources/state_compaction_test.go index 24e4af61ebc..cc9698de3ad 100644 --- a/bundle/direct/dresources/state_compaction_test.go +++ b/bundle/direct/dresources/state_compaction_test.go @@ -1,6 +1,7 @@ package dresources import ( + "encoding/json" "strings" "testing" @@ -10,6 +11,40 @@ import ( "github.com/stretchr/testify/require" ) +const ( + // largeDashboard is a serialized_dashboard whose JSON encoding exceeds + // stateHashPlaceholderLen, so it is actually compacted. Shared by the tests that assert + // hashing happens, since a shorter value is deliberately left raw. + largeDashboard = `{"pages":[{"name":"p1","displayName":"Page One","layout":[{"widget":{"name":"w1"}}]}]}` + + // smallDashboard is a serialized_dashboard whose JSON encoding fits within + // stateHashPlaceholderLen, so hashing it would grow the state and it is persisted raw. + smallDashboard = `{"pages":[{"name":"p1"}]}` +) + +// requireLargeEnoughToHash asserts a fixture is on the hashed side of the size threshold. +// Whether a value is hashed depends on the length of its JSON encoding, so shrinking a +// fixture would silently leave it raw and make the tests below assert the opposite of what +// they were written for. Fail with an explicit message instead. +func requireLargeEnoughToHash(t *testing.T, content string) { + t.Helper() + encoded, err := json.Marshal(content) + require.NoError(t, err) + require.Greater(t, len(encoded), stateHashPlaceholderLen, + "fixture must encode to more than %d bytes to be hashed; enlarge it", stateHashPlaceholderLen) +} + +// requireTooSmallToHash is the inverse of requireLargeEnoughToHash: it asserts a fixture +// stays on the raw side of the threshold, so enlarging it cannot silently turn a +// persisted-raw test into a hashing test that passes for the wrong reason. +func requireTooSmallToHash(t *testing.T, content string) { + t.Helper() + encoded, err := json.Marshal(content) + require.NoError(t, err) + require.LessOrEqual(t, len(encoded), stateHashPlaceholderLen, + "fixture must encode to at most %d bytes to be persisted raw; shrink it", stateHashPlaceholderLen) +} + // TestHashedInStateFieldsAreTopLevel guards the shallow-copy assumption in CompactState: // every hashed_in_state path declared in resources.yml must be a top-level field. A nested // path would be mutated through memory shared with the deploy value (see CompactState), so @@ -54,9 +89,10 @@ func TestCompactStateNoDeclaredFields(t *testing.T) { // same hash, so a diff computed after hashing-on-read shows no spurious change and // the next save rewrites the state compactly. func TestCompactStateMigratesLegacyFullContent(t *testing.T) { - content := `{"pages":[{"name":"p1"}]}` - legacy := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: content}} - config := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: content}} + requireLargeEnoughToHash(t, largeDashboard) + + legacy := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: largeDashboard}} + config := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: largeDashboard}} cfg := GetResourceConfig("dashboards") compactedLegacy, err := CompactState(cfg, legacy) @@ -73,16 +109,18 @@ func TestCompactStateMigratesLegacyFullContent(t *testing.T) { // a stable placeholder: the same content always hashes to the same value and // different content differs. func TestHashStateValue(t *testing.T) { - stringHash, err := hashStateValue("hello") + requireLargeEnoughToHash(t, largeDashboard) + + stringHash, err := hashStateValue(largeDashboard) require.NoError(t, err) require.IsType(t, "", stringHash) assert.True(t, strings.HasPrefix(stringHash.(string), stateHashPrefix)) - again, err := hashStateValue("hello") + again, err := hashStateValue(largeDashboard) require.NoError(t, err) assert.Equal(t, stringHash, again) - other, err := hashStateValue("world") + other, err := hashStateValue(strings.Replace(largeDashboard, "p1", "p2", 1)) require.NoError(t, err) assert.NotEqual(t, stringHash, other) } @@ -90,7 +128,9 @@ func TestHashStateValue(t *testing.T) { // TestHashStateValueIdempotent verifies re-hashing an existing placeholder returns it // unchanged, so re-compacting an already-compact state does not double-hash. func TestHashStateValueIdempotent(t *testing.T) { - hashed, err := hashStateValue("some content") + requireLargeEnoughToHash(t, largeDashboard) + + hashed, err := hashStateValue(largeDashboard) require.NoError(t, err) again, err := hashStateValue(hashed) @@ -98,6 +138,72 @@ func TestHashStateValueIdempotent(t *testing.T) { assert.Equal(t, hashed, again) } +// TestHashStateValueSkipsSmallValues verifies a value whose JSON encoding would not +// shrink is left raw, so state never grows to hold a placeholder longer than the content +// it replaces. The boundary cases pin the comparison to the JSON encoding (which adds the +// surrounding quotes for a string) rather than the raw string length. +func TestHashStateValueSkipsSmallValues(t *testing.T) { + // A string encodes with two surrounding quotes, so this is exactly at the limit. + atLimit := strings.Repeat("x", stateHashPlaceholderLen-2) + out, err := hashStateValue(atLimit) + require.NoError(t, err) + assert.Equal(t, atLimit, out) + + overLimit := atLimit + "x" + out, err = hashStateValue(overLimit) + require.NoError(t, err) + assert.True(t, strings.HasPrefix(out.(string), stateHashPrefix)) + assert.Len(t, out, stateHashPlaceholderLen) +} + +// TestCompactStateSkipsSmallField verifies the size check reaches CompactState, so a +// small field stays raw in the state it saves and on every side of the diff it compacts. +// Persisting it raw is what keeps the state smaller than a hash placeholder would. +func TestCompactStateSkipsSmallField(t *testing.T) { + requireTooSmallToHash(t, smallDashboard) + + state := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: smallDashboard}} + + cfg := GetResourceConfig("dashboards") + out, err := CompactState(cfg, state) + require.NoError(t, err) + assert.Equal(t, smallDashboard, out.(*DashboardState).SerializedDashboard) + + // The persisted form holds the content itself, and holding it costs less than the + // placeholder that replacing it would have written. + persisted, err := json.Marshal(out.(*DashboardState).SerializedDashboard) + require.NoError(t, err) + assert.LessOrEqual(t, len(persisted), stateHashPlaceholderLen) + + // Compacting the already-raw state again leaves it alone, so repeated saves and every + // side of the diff keep comparing content against content. + out2, err := CompactState(cfg, out) + require.NoError(t, err) + assert.Equal(t, smallDashboard, out2.(*DashboardState).SerializedDashboard) +} + +// TestCompactStateHashesLargeField is the counterpart of TestCompactStateSkipsSmallField: +// once the content outgrows a placeholder, compaction replaces it and the state shrinks. +func TestCompactStateHashesLargeField(t *testing.T) { + requireLargeEnoughToHash(t, largeDashboard) + + state := &DashboardState{DashboardConfig: resources.DashboardConfig{SerializedDashboard: largeDashboard}} + + out, err := CompactState(GetResourceConfig("dashboards"), state) + require.NoError(t, err) + compacted := out.(*DashboardState).SerializedDashboard + require.IsType(t, "", compacted) + assert.True(t, strings.HasPrefix(compacted.(string), stateHashPrefix)) + assert.Len(t, compacted, stateHashPlaceholderLen) + + // The whole point of hashing: the persisted form is smaller than the raw content. + persisted, err := json.Marshal(compacted) + require.NoError(t, err) + raw, err := json.Marshal(largeDashboard) + require.NoError(t, err) + assert.Less(t, len(persisted), len(raw)) +} + // TestHashStateValueEmptyAndNil verifies empty and nil values pass through unchanged, // since there is nothing to hash. func TestHashStateValueEmptyAndNil(t *testing.T) { From 88f86f008f45c039c579680d1ac8324465198477 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 15:07:37 +0000 Subject: [PATCH 23/27] update acceptance tests with new hashing rules --- .../dashboard/recreation/out.state_after_bind.direct.json | 2 +- .../dashboard-state-sha/out.create.serialized.json | 2 +- .../dashboard-state-sha/out.plan_create.direct.json | 2 +- .../resources/dashboard-state-sha/out.plan_skip.direct.json | 2 +- .../dashboard-state-sha/out.plan_update.direct.json | 4 ++-- .../dashboard-state-sha/out.update.serialized.json | 2 +- .../resources/dashboards/detect-change/out.plan.direct.json | 2 +- .../bundle/resources/dashboards/simple/out.plan.direct.json | 6 +++--- .../dashboards/unpublish-out-of-band/out.plan.direct.json | 6 +++--- 9 files changed, 14 insertions(+), 14 deletions(-) diff --git a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json index c52412ab5fb..eb3a71d91ff 100644 --- a/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json +++ b/acceptance/bundle/deployment/bind/dashboard/recreation/out.state_after_bind.direct.json @@ -12,7 +12,7 @@ "etag": [ETAG], "parent_path": "/Workspace/Users/[USERNAME]/.bundle/test-bundle-[UNIQUE_NAME]/default/resources", "published": true, - "serialized_dashboard": "sha256_hashed_in_state:[HASH]", + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Page One\",\"name\":\"02724bf2\"}]}", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" } } diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json b/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json index c21b142583a..e054033f0ad 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.create.serialized.json @@ -1,3 +1,3 @@ [ - "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n" + "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}]}]}\n" ] diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json index b9d477135b6..43e9be8ba00 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_create.direct.json @@ -10,7 +10,7 @@ "embed_credentials": true, "parent_path": "/Workspace/Users/[USERNAME]", "published": true, - "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\"}]}\n", + "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}]}]}\n", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" } } diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json index 595950529ab..af7e1c48f18 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_skip.direct.json @@ -16,7 +16,7 @@ "parent_path": "/Workspace/Users/[USERNAME]", "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json", "published": true, - "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}],\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", "update_time": "[TIMESTAMP]", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" }, diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json b/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json index f6b9a4efa37..ef60e06b05c 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.plan_update.direct.json @@ -12,7 +12,7 @@ "embed_credentials": true, "parent_path": "/Workspace/Users/[USERNAME]", "published": true, - "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n", + "serialized_dashboard": "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}]}]}\n", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" } }, @@ -26,7 +26,7 @@ "parent_path": "/Workspace/Users/[USERNAME]", "path": "/Users/[USERNAME]/dashboard-state-sha [UUID].lvdash.json", "published": true, - "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", + "serialized_dashboard": "{\"pages\":[{\"displayName\":\"Sales Overview\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}],\"name\":\"main\",\"pageType\":\"PAGE_TYPE_CANVAS\"}]}\n", "update_time": "[TIMESTAMP]", "warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]" }, diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json b/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json index 5c8339e26b5..10516eb9e66 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json +++ b/acceptance/bundle/resources/dashboard-state-sha/out.update.serialized.json @@ -1,3 +1,3 @@ [ - "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\"}]}\n" + "{\"pages\":[{\"name\":\"main\",\"displayName\":\"Sales Overview v2\",\"layout\":[{\"widget\":{\"name\":\"counter1\"}}]}]}\n" ] diff --git a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json index e80adf8c285..b7b6753f895 100644 --- a/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/detect-change/out.plan.direct.json @@ -67,7 +67,7 @@ "reason": "etag_based", "old": "sha256_hashed_in_state:[HASH][0]", "new": "sha256_hashed_in_state:[HASH][0]", - "remote": "sha256_hashed_in_state:[HASH][1]" + "remote": "{}\n" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json index 32fe8c1b420..5e6ad9eb0a6 100644 --- a/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/simple/out.plan.direct.json @@ -50,9 +50,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:[HASH][0]", - "new": "sha256_hashed_in_state:[HASH][0]", - "remote": "sha256_hashed_in_state:[HASH][1]" + "old": "{ }\n", + "new": "{ }\n", + "remote": "{}\n" }, "update_time": { "action": "skip", diff --git a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json index 0197870475c..21753c21f0e 100644 --- a/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json +++ b/acceptance/bundle/resources/dashboards/unpublish-out-of-band/out.plan.direct.json @@ -66,9 +66,9 @@ "serialized_dashboard": { "action": "skip", "reason": "etag_based", - "old": "sha256_hashed_in_state:[HASH][0]", - "new": "sha256_hashed_in_state:[HASH][0]", - "remote": "sha256_hashed_in_state:[HASH][1]" + "old": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}", + "new": "{\"pages\":[{\"displayName\":\"Test Dashboard\",\"name\":\"test-page\"}]}", + "remote": "sha256_hashed_in_state:[HASH]" }, "update_time": { "action": "skip", From 6f2ca0707d62251fce48da018a93b9f2e1267c12 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 15:14:47 +0000 Subject: [PATCH 24/27] decrease comments --- bundle/direct/dresources/state_compaction.go | 6 ++---- bundle/direct/dresources/state_compaction_test.go | 3 +-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/bundle/direct/dresources/state_compaction.go b/bundle/direct/dresources/state_compaction.go index 6e230d55fdb..055a86d821d 100644 --- a/bundle/direct/dresources/state_compaction.go +++ b/bundle/direct/dresources/state_compaction.go @@ -27,9 +27,7 @@ const stateHashPlaceholderLen = len(stateHashPrefix) + sha256.Size*2 // serialized_dashboard) compactly in state instead of their full contents. // // It is idempotent and stable: nil, an empty string, a value that is already a -// placeholder, and a value too small to be worth hashing are returned unchanged, so -// re-compacting an already-compact state and comparing a fresh config against stored -// state both behave predictably. +// placeholder, and a value too small to be worth hashing are returned unchanged func hashStateValue(v any) (any, error) { if s, ok := v.(string); ok { if s == "" || strings.HasPrefix(s, stateHashPrefix) { @@ -41,7 +39,7 @@ func hashStateValue(v any) (any, error) { return v, nil } - // json.Marshal sorts map keys, so the digest is stable across runs for equal values. + // json.Marshal sorts map keys data, err := json.Marshal(v) if err != nil { return nil, fmt.Errorf("marshalling value for hashing: %w", err) diff --git a/bundle/direct/dresources/state_compaction_test.go b/bundle/direct/dresources/state_compaction_test.go index cc9698de3ad..5f171f72fe4 100644 --- a/bundle/direct/dresources/state_compaction_test.go +++ b/bundle/direct/dresources/state_compaction_test.go @@ -13,8 +13,7 @@ import ( const ( // largeDashboard is a serialized_dashboard whose JSON encoding exceeds - // stateHashPlaceholderLen, so it is actually compacted. Shared by the tests that assert - // hashing happens, since a shorter value is deliberately left raw. + // stateHashPlaceholderLen, so it is actually compacted. largeDashboard = `{"pages":[{"name":"p1","displayName":"Page One","layout":[{"widget":{"name":"w1"}}]}]}` // smallDashboard is a serialized_dashboard whose JSON encoding fits within From 8263e7b066183e08d3b299453d3e91a0410462b6 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Thu, 6 Aug 2026 15:35:24 +0000 Subject: [PATCH 25/27] add periods to comments --- bundle/direct/dresources/state_compaction.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bundle/direct/dresources/state_compaction.go b/bundle/direct/dresources/state_compaction.go index 055a86d821d..02cb4fbaafe 100644 --- a/bundle/direct/dresources/state_compaction.go +++ b/bundle/direct/dresources/state_compaction.go @@ -27,7 +27,7 @@ const stateHashPlaceholderLen = len(stateHashPrefix) + sha256.Size*2 // serialized_dashboard) compactly in state instead of their full contents. // // It is idempotent and stable: nil, an empty string, a value that is already a -// placeholder, and a value too small to be worth hashing are returned unchanged +// placeholder, and a value too small to be worth hashing are returned unchanged. func hashStateValue(v any) (any, error) { if s, ok := v.(string); ok { if s == "" || strings.HasPrefix(s, stateHashPrefix) { @@ -39,7 +39,7 @@ func hashStateValue(v any) (any, error) { return v, nil } - // json.Marshal sorts map keys + // json.Marshal sorts map keys, so the digest is stable across runs for equal values. data, err := json.Marshal(v) if err != nil { return nil, fmt.Errorf("marshalling value for hashing: %w", err) From 55e9d268e585f0e3b8963c7b43563f6af4a0ac89 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Fri, 7 Aug 2026 09:38:57 +0000 Subject: [PATCH 26/27] docs: state the real hashed_in_state invariant and downgrade behaviour The size gate means a small value is stored raw on every side, so the old wording ("all three sides must be hashes") no longer holds. Restate the real invariant: CompactState is a deterministic pure function of the value, so equal values always compact identically and the diff comparisons stay meaningful. Also document the downgrade case, where an older CLI reads a stored hash as contents and republishes once (non-destructive, self-correcting). Co-authored-by: Isaac --- bundle/direct/dresources/README.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/bundle/direct/dresources/README.md b/bundle/direct/dresources/README.md index c0b54413f3b..cb366f2c348 100644 --- a/bundle/direct/dresources/README.md +++ b/bundle/direct/dresources/README.md @@ -55,15 +55,17 @@ on field behaviour might result in recreate. See dstate/migrate.go on how to han ## hashed_in_state: storing large fields as content hashes -Declare a field under `hashed_in_state` in `resources.yml` when it holds large content that is only ever compared for equality and never read back from state. The engine then persists only a `sha256_hashed_in_state:` content hash for that field (via `CompactState`), and — crucially — hashes it on **every** value entering the diff: saved state, the local config, and the remapped remote. Once the saved value is a hash, all three sides must be hashes or the comparisons (`Old==New` for a local change, `Remote==New` for remote drift) would be hash-vs-content nonsense. The full contents stay only in the plan's `new_state` and are sent to the API on every deploy, so the deploy is unaffected. +Declare a field under `hashed_in_state` in `resources.yml` when it holds large content that is only ever compared for equality and never read back from state. The engine then persists only a `sha256_hashed_in_state:` content hash for that field (via `CompactState`), and applies the same transform to **every** value entering the diff: saved state, the local config, and the remapped remote. Correctness rests on `CompactState` being a deterministic pure function of the value — equal values always compact to the same thing — so the comparisons (`Old==New` for a local change, `Remote==New` for remote drift) stay meaningful: two equal values compact to two equal hashes, two different values to two different hashes. The full contents stay only in the plan's `new_state` and are sent to the API on every deploy, so the deploy is unaffected. A field qualifies if it is large and is never read back from state by any code path (e.g. not consumed raw by `OverrideChangeDesc` or by state export). Hashing is skipped when it would not pay for itself: a value whose JSON encoding is no longer than the 87-byte placeholder is persisted as is. The verdict depends only on the value, so the state being saved and all three sides of the diff agree on it and stay comparable — including when the same field is small for one resource and large for another, or grows past the threshold between deploys. -**`hashed_in_state` is orthogonal to `ignore_remote_changes`.** Because the remote side is hashed too, remote drift on a hashed field is detected as `hash != hash` — so a field can be `hashed_in_state` *without* being `ignore_remote_changes` (as long as the server echoes the value back unchanged). The two are declared independently. +**`hashed_in_state` is orthogonal to `ignore_remote_changes`.** Because the remote side goes through the same transform, remote drift on a hashed field is detected as `hash != hash` — so a field can be `hashed_in_state` *without* being `ignore_remote_changes` (as long as the server echoes the value back unchanged). The two are declared independently. -`dashboards.serialized_dashboard` happens to need both, for unrelated reasons: it is `hashed_in_state` because the inlined dashboard JSON is large, and it is *separately* `ignore_remote_changes` because the **server normalizes** it (adds `pageType`, reorders keys) so its remote hash never equals the config hash — drift is detected via `etag` instead. The plan therefore reports the field as a hash on all three sides. No state version bump is needed: legacy full-content state is hashed on read for comparison and rewritten compactly on the next save. +`dashboards.serialized_dashboard` happens to need both, for unrelated reasons: it is `hashed_in_state` because the inlined dashboard JSON is large, and it is *separately* `ignore_remote_changes` because the **server normalizes** it (adds `pageType`, reorders keys) so its remote hash never equals the config hash — drift is detected via `etag` instead. + +No state version bump is needed for reading forward: legacy full-content state is compacted on read for comparison and rewritten compactly on the next save. A downgrade is the reverse case — an older CLI that predates `hashed_in_state` reads the stored hash as if it were the contents; since `serialized_dashboard` is not `ignore_local_changes`, the `Old(hash) != New(contents)` mismatch is read as a local change and republishes the dashboard once. That is non-destructive and self-correcting (the old CLI rewrites full contents; a later upgrade re-hashes on read), so no version gate is imposed against it. ## RemapState is a dumb copy; DoRead owns all remapping From 24c80921466987c8a5f57ef0f2fc0155027d0d86 Mon Sep 17 00:00:00 2001 From: Sankalp-Mittal Date: Tue, 11 Aug 2026 10:41:26 +0000 Subject: [PATCH 27/27] acc: drop stale Local key from dashboard-state-sha test.toml The `Local` acceptance-config field was dropped when main was merged into this branch (main never had it), but test.toml still set `Local = true`. The framework rejects unknown test.toml keys, so dashboard-state-sha failed on every platform with "Undecoded key: Local". `Cloud = false` already expresses the local-only gating, so just remove the dangling key and regenerate its out.test.toml snapshot. Co-authored-by: Isaac --- acceptance/bundle/resources/dashboard-state-sha/out.test.toml | 1 - acceptance/bundle/resources/dashboard-state-sha/test.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/acceptance/bundle/resources/dashboard-state-sha/out.test.toml b/acceptance/bundle/resources/dashboard-state-sha/out.test.toml index 71970b719d4..57b0f616850 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/out.test.toml +++ b/acceptance/bundle/resources/dashboard-state-sha/out.test.toml @@ -1,4 +1,3 @@ -Local = true Cloud = false EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] EnvMatrix.READPLAN = ["", "1"] diff --git a/acceptance/bundle/resources/dashboard-state-sha/test.toml b/acceptance/bundle/resources/dashboard-state-sha/test.toml index 6b239a11106..577424e032a 100644 --- a/acceptance/bundle/resources/dashboard-state-sha/test.toml +++ b/acceptance/bundle/resources/dashboard-state-sha/test.toml @@ -5,7 +5,6 @@ # content hash), which the in-memory testserver covers fully and deterministically. On real # cloud the read-after-deploy plan/state reads hit dashboard eventual consistency, and unlike # the tests under dashboards/ this dir doesn't inherit their retry/INJECT_STALE machinery. -Local = true Cloud = false # dashboard.lvdash.json is rendered from dashboard.tmpl in the script (re-rendered for the edit).