diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index 4f39093beb3..33b751f8601 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -408,6 +408,72 @@ jobs: run: | go tool -modfile=tools/task/go.mod task test-sandbox + test-fuzz: + needs: + - cleanups + + # Wide rotating seed window with drift on: too slow for every PR, so nightly only and + # not part of test-result. The committed acceptance test still checks no-panic per PR. + if: ${{ github.event_name == 'schedule' }} + name: "task test-fuzz" + runs-on: + group: databricks-protected-runner-group-large + labels: linux-ubuntu-latest-large + + defaults: + run: + shell: bash + + permissions: + id-token: write + contents: read + # For the failure-reporting step's PR comment. + pull-requests: write + + steps: + - name: Checkout repository and submodules + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + - name: Setup build environment + uses: ./.github/actions/setup-build-environment + with: + cache-key: test-fuzz + + - name: Run tests + env: + FUZZ_SEED_COUNT: "25" + run: | + # Non-overlapping windows, so CI explores new configs every run. + export FUZZ_SEED_START=$(( GITHUB_RUN_NUMBER * FUZZ_SEED_COUNT )) + go tool -modfile=tools/task/go.mod task test-fuzz + + # Not in test-result, so surface failures by commenting on the PR under test. + - name: Report failure + if: ${{ failure() }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + COMMIT: ${{ github.sha }} + run: | + body=$(cat < ...\`. + EOF + ) + + # The commit's pulls endpoint returns the PR that introduced it. + pr=$(gh api "repos/$GITHUB_REPOSITORY/commits/$COMMIT/pulls" --jq '.[0].number // empty') + if [ -n "$pr" ]; then + gh pr comment "$pr" --body "$body" + else + echo "No PR found for commit $COMMIT; skipping failure comment" >&2 + fi + # This job groups the result of all the above test jobs. # It is a required check, so it blocks auto-merge and the merge queue. # diff --git a/.gitignore b/.gitignore index 4b82c6d1521..0403b5b21e4 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ *.log coverage.txt coverage-acceptance.txt +coverage-fuzz.txt .coverage __pycache__ diff --git a/Taskfile.yml b/Taskfile.yml index 1eb41d6ac53..8878c2f9950 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -730,6 +730,53 @@ tasks: --packages ./acceptance/... \ -- -timeout=${LOCAL_TIMEOUT:-60m} -run "TestAccept/cmd/sandbox" + test-fuzz: + desc: Run schema fuzz invariant tests (random configs, direct engine) + # No `sources:` fingerprint: the window depends on FUZZ_* env vars Task can't see. + cmds: + - | + # Wider window than the committed run, with drift on; a repro narrows it via + # FUZZ_SEED_START/COUNT. + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-200}" + export FUZZ_CHECK_DRIFT="${FUZZ_CHECK_DRIFT:-1}" + # -count=1: only the script reads FUZZ_*, so the test cache would serve another window's + # result as this one's. + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./acceptance/... \ + -- -count=1 -timeout=${LOCAL_TIMEOUT:-30m} -run "TestAccept/bundle/invariant/fuzz" + + test-fuzz-cover: + desc: Run the schema fuzzer under coverage and report which CLI packages it exercises + # No `sources:` fingerprint: like test-fuzz, the window depends on FUZZ_* env vars. + cmds: + - rm -fr ./acceptance/build/cover-fuzz/ ./acceptance/build/cover-fuzz-merged/ + - mkdir -p ./acceptance/build/cover-fuzz-merged/ + - | + # CLI_GOCOVERDIR makes the harness build a -cover CLI and set GOCOVERDIR per run, so every + # fuzzed `bundle` invocation drops aggregatable counter files. Drift off so the run + # exercises the full deploy/plan path instead of stopping on the first drift. + export CLI_GOCOVERDIR=build/cover-fuzz + export FUZZ_SEED_COUNT="${FUZZ_SEED_COUNT:-100}" + unset FUZZ_CHECK_DRIFT + {{.GO_TOOL}} gotestsum \ + --format ${GOTESTSUM_FORMAT:-pkgname-and-test-fails} \ + --no-summary=skipped \ + --packages ./acceptance/... \ + -- -count=1 -timeout=${LOCAL_TIMEOUT:-40m} -run "TestAccept/bundle/invariant/fuzz" || true + - "go tool covdata merge -i $(printf '%s,' acceptance/build/cover-fuzz/* | sed 's/,$//') -o acceptance/build/cover-fuzz-merged/" + - go tool covdata textfmt -i acceptance/build/cover-fuzz-merged -o coverage-fuzz.txt + - | + echo "== total CLI coverage exercised by the fuzz run ==" + go tool cover -func=coverage-fuzz.txt | awk '/^total:/{print $NF}' + echo + echo "== bundle/cmd packages by coverage (ascending; 0.0% = never exercised) ==" + go tool covdata percent -i=acceptance/build/cover-fuzz-merged \ + | awk '/coverage:/{p=$3; gsub(/%/,"",p); sub("github.com/databricks/cli/","",$1); printf "%6.1f%% %s\n", p, $1}' \ + | grep -E ' (bundle|cmd/bundle)/' \ + | sort -n + # --- Integration tests --- integration: diff --git a/acceptance/bin/emit_fuzz_config.py b/acceptance/bin/emit_fuzz_config.py new file mode 100755 index 00000000000..bef3659861d --- /dev/null +++ b/acceptance/bin/emit_fuzz_config.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +""" +Emit a fuzz databricks.yml on stdout for the current seed, picking the strategy from FUZZ_MODE so +the invariant scripts don't each duplicate the branch: + + generate (default) - build from scratch by walking `bundle schema` (gen_fuzz_config.py). + mutate - perturb a curated invariant config (mutate_fuzz_config.py). + +Reads its inputs from the environment the invariant scripts export: FUZZ_SEED, FUZZ_SCHEMA, +UNIQUE_NAME, TESTDIR. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from envsubst import substitute_variables +from gen_fuzz_config import gen_config, to_yaml +from mutate_fuzz_config import load_yaml, mutate + +# Curated single-resource configs that deploy standalone (only $UNIQUE_NAME, no init script). All +# are in the invariant INPUT_CONFIG matrix, so they stay deploy-verified. +MUTATE_BASES = [ + "catalog", + "external_location", + "job", + "model", + "model_serving_endpoint", + "pipeline", + "registered_model", + "schema", + "secret_scope", + "sql_warehouse", + "volume", +] + + +def generate(seed): + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + return to_yaml(gen_config(schema, seed, os.environ["UNIQUE_NAME"])) + + +def mutate_base(seed): + name = MUTATE_BASES[seed % len(MUTATE_BASES)] + path = os.path.join(os.environ["TESTDIR"], "..", "configs", name + ".yml.tmpl") + unique = os.environ["UNIQUE_NAME"] + with open(path) as f: + rendered = substitute_variables(f.read()) + config = load_yaml(rendered) + # The schema lets mutate inject valid optional fields, not just perturb existing ones. + with open(os.environ["FUZZ_SCHEMA"]) as f: + schema = json.load(f) + return to_yaml(mutate(config, seed, schema=schema, unique=unique)) + + +def main(): + seed = int(os.environ["FUZZ_SEED"]) + mode = os.environ.get("FUZZ_MODE", "generate") + if mode == "generate": + sys.stdout.write(generate(seed)) + elif mode == "mutate": + sys.stdout.write(mutate_base(seed)) + else: + sys.exit(f"emit_fuzz_config: unknown FUZZ_MODE {mode!r}") + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/gen_fuzz_config.py b/acceptance/bin/gen_fuzz_config.py new file mode 100755 index 00000000000..e4aecd24aab --- /dev/null +++ b/acceptance/bin/gen_fuzz_config.py @@ -0,0 +1,427 @@ +""" +Generate a random bundle config from the bundle JSON schema. + +Walks `databricks bundle schema` (resolving $ref, picking concrete oneOf/anyOf branches) and emits +one random resource, seeded by the caller. Free-form scalars are sometimes replaced with dangerous +values (DANGEROUS_STRINGS/INTS) to probe input handling. The harness drops configs the CLI +rejects, so output may be structurally random but invalid. + +Used as a library by emit_fuzz_config.py and mutate_fuzz_config.py. +""" + +import json +import os +import random +import re +import sys + +# The schema is recursive (e.g. task -> for_each_task -> task); cap the walk. +MAX_DEPTH = 6 + +# The ${...} interpolation branch the schema wraps every field in (see +# bundle/internal/schema/main.go addInterpolationPatterns); we emit concrete values. +INTERPOLATION_MARKER = "\\$\\{" + +# Keep in sync with libs/jsonschema.Type. gen_scalar exits on anything else. +SCALAR_TYPES = {"boolean", "integer", "number", "string"} + +# Cross-resource refs must resolve on every workspace (fake server and real UC). "main"/"default" +# are the standard seeded catalog/schema; a random name deploys on the fake server but real UC +# rejects it (CATALOG_DOES_NOT_EXIST), dropping the config. +DEFAULT_CATALOG = "main" +DEFAULT_SCHEMA = "default" + +# "account users" exists on every workspace; each securable gets one privilege UC accepts. A random +# principal or inapplicable privilege deploys on the fake server but fails on UC. +DEFAULT_PRINCIPAL = "account users" +GRANT_PRIVILEGE = { + "catalogs": "USE_CATALOG", + "schemas": "USE_SCHEMA", + "volumes": "READ_VOLUME", + "registered_models": "EXECUTE", + "external_locations": "READ_FILES", + "vector_search_indexes": "SELECT", +} + +# Permissions can't be variable refs; each entry needs a concrete principal and a level valid for +# the resource type. +DEFAULT_PERMISSION_GROUP = "users" +PERMISSION_LEVEL = { + "alerts": "CAN_MANAGE", + "apps": "CAN_USE", + "clusters": "CAN_ATTACH_TO", + "dashboards": "CAN_READ", + "database_instances": "CAN_USE", + "experiments": "CAN_READ", + "genie_spaces": "CAN_READ", + "jobs": "CAN_VIEW", + "model_serving_endpoints": "CAN_VIEW", + "models": "CAN_READ", + "pipelines": "CAN_VIEW", + "postgres_projects": "CAN_USE", + "secret_scopes": "READ", + "sql_warehouses": "CAN_VIEW", + "vector_search_endpoints": "CAN_USE", +} + +# Fields the backend computes; emitting them causes false drift after migrate. Mirrors +# output_only/backend_defaults in dresources/resources.yml. Blocked by name everywhere, so writable +# exceptions (an external volume's storage_location) need a curated config. +SKIP_PROPERTY_NAMES = frozenset( + { + "browse_only", + "created_at", + "created_by", + "creator_name", + # Backend-assigned; the CLI rejects an etag set in bundle config. + "etag", + "full_name", + "metastore_id", + "owner", + "storage_location", + "updated_at", + "updated_by", + } +) + +# Fields these resources need to deploy but that the schema's required[] omits (or can't express in +# YAML). Values come from the *_BY_RESOURCE tables below. +RESOURCE_REQUIRED_FIELDS = { + "registered_models": frozenset({"catalog_name", "name", "schema_name"}), + "dashboards": frozenset({"display_name", "file_path", "warehouse_id"}), + "alerts": frozenset({"display_name", "file_path", "warehouse_id"}), + "apps": frozenset({"name", "source_code_path"}), + "genie_spaces": frozenset({"serialized_space", "title", "warehouse_id"}), +} + +# Fields that conflict with the set we emit. Dashboards/Genie spaces take their body from file_path +# XOR an inline serialized_* field; emitting both is rejected. +RESOURCE_SKIP_FIELDS = { + "dashboards": frozenset({"serialized_dashboard"}), + "genie_spaces": frozenset({"file_path"}), + "apps": frozenset({"git_repository", "git_source"}), +} + +# Resources allowing only a fixed field set in YAML. Alerts read their spec from the +# .dbalert.json at file_path; the CLI rejects other fields (load_dbalert_files.go). +RESOURCE_FIELD_ALLOWLIST = { + "alerts": frozenset({"display_name", "file_path", "lifecycle", "permissions", "warehouse_id"}), +} + +# Serialized-body fixtures copied into each seed dir from invariant/data; the extension selects the +# parser. +FILE_PATH_BY_RESOURCE = { + "dashboards": "./dashboard.lvdash.json", + "alerts": "./alert.dbalert.json", +} + +# A local directory holding app source, also copied in from data/. +APP_SOURCE_CODE_PATH = "./app" + +# An absolute workspace path is treated as already-remote, skipping the local-notebook +# existence/extension check a bare token would fail. +NOTEBOOK_PATH = "/Shared/notebook" + +# parent_path is a workspace folder; pin it to a valid one. The CLI re-adds the /Workspace prefix +# on read, so a mismatched value plans a spurious recreate. +PARENT_PATH = "/Workspace/Shared" + +# String in the schema but parsed as protobuf.Duration at load (suspend_timeout_duration, ttl); a +# bare token fails to parse. +DURATION_VALUE = "3600s" + +# Dangerous/near-range-end probes for free-form scalars. The CLI must reject or round-trip these +# without panicking; mutate_fuzz_config reuses them. +DANGEROUS_STRINGS = [ + "", + " ", + "a" * 300, + "line1\nline2", + "tab\there", + "\U0001f680-unicode-\u00e9", + "quote\"and'apostrophe", + "${resources.jobs.does_not_exist.id}", + "../../etc/passwd", +] +DANGEROUS_INTS = [ + 2**31 - 1, + 2**31, + -(2**31), + 2**63 - 1, + -(2**63), + -1, +] + +# Inject a dangerous value only sometimes, so the config usually still deploys and exercises the +# invariant, not just the reject path. +DANGEROUS_PROB = 0.15 + + +class Generator: + def __init__(self, schema, rng, unique): + self.root = schema + self.rng = rng + self.unique = unique + # Top-level resource type, set before generating its element so grants/permissions can pick + # a value valid for that securable. + self.rtype = None + + def resolve(self, schema): + # Follow $ref chains ("#/$defs/.../resources.Job"), indexing $defs by path segment. + while isinstance(schema, dict) and "$ref" in schema: + cur = self.root["$defs"] + for part in schema["$ref"].split("/")[2:]: + cur = cur[part] + schema = cur + return schema + + def is_interpolation(self, branch): + return branch.get("type") == "string" and INTERPOLATION_MARKER in branch.get("pattern", "") + + def choose_branch(self, branches): + # Prefer concrete branches over the ${...} alternatives. + concrete = [b for b in branches if not self.is_interpolation(b)] + return self.rng.choice(concrete or branches) + + def field_behaviors(self, schema): + if not isinstance(schema, dict): + return [] + resolved = self.resolve(schema) + behaviors = list(schema.get("x-databricks-field-behaviors", [])) + if resolved is not schema: + behaviors.extend(resolved.get("x-databricks-field-behaviors", [])) + return behaviors + + def should_skip_property(self, prop_name, prop_schema): + if prop_name in SKIP_PROPERTY_NAMES: + return True + if prop_name in RESOURCE_SKIP_FIELDS.get(self.rtype, ()): + return True + resolved = self.resolve(prop_schema) + if "OUTPUT_ONLY" in self.field_behaviors(prop_schema): + return True + if resolved.get("readOnly"): + return True + return False + + def gen(self, schema, depth, name=""): + # A Genie space body is free-form but the backend rejects unknown keys, so emit the minimal + # accepted body instead of a random object. + if name == "serialized_space": + return {"version": 1} + + schema = self.resolve(schema) + if not isinstance(schema, dict) or not schema: + return self.gen_scalar({"type": "string"}, name) + + if name == "grants": + return self.gen_grants() + if name == "permissions": + return self.gen_permissions() + + if "const" in schema: + return schema["const"] + if schema.get("enum"): + return self.rng.choice(schema["enum"]) + + for key in ("oneOf", "anyOf"): + if schema.get(key): + return self.gen(self.choose_branch(schema[key]), depth, name) + + t = schema.get("type") + if t == "object" or "properties" in schema or self.is_map(schema): + return self.gen_object(schema, depth) + if t == "array": + return self.gen_array(schema, depth, name) + return self.gen_scalar(schema, name) + + def is_map(self, schema): + return isinstance(schema.get("additionalProperties"), dict) and not schema.get("properties") + + def gen_object(self, schema, depth): + props = schema.get("properties", {}) + required = set(schema.get("required", [])) + allowlist = None + if depth == 0 and self.rtype: + required |= RESOURCE_REQUIRED_FIELDS.get(self.rtype, set()) + allowlist = RESOURCE_FIELD_ALLOWLIST.get(self.rtype) + result = {} + + for prop_name, prop_schema in props.items(): + # A restricted resource (e.g. alerts) rejects any field outside its allow-list, even a + # schema-required one it reads from the file instead. + if allowlist is not None and prop_name not in allowlist: + continue + if self.should_skip_property(prop_name, prop_schema): + continue + # Emit optional fields less often deeper down to keep configs from exploding. + keep = prop_name in required or (depth < MAX_DEPTH and self.rng.random() < 0.35) + if not keep: + continue + value = self.gen(prop_schema, depth + 1, prop_name) + # Drop an object whose every property was skipped: `{}` carries no information and some + # fields reject it outright. + if value is None or value == {}: + continue + result[prop_name] = value + + # Map type: synthesize a few random keys, e.g. resources. or string maps like tags. + if self.is_map(schema): + for _ in range(self.rng.randint(1, 2)): + key = self.token() + result[key] = self.gen(schema["additionalProperties"], depth + 1, key) + + return result + + def gen_array(self, schema, depth, name): + items = schema.get("items") + if not items or depth >= MAX_DEPTH: + return [] + return [self.gen(items, depth + 1, name) for _ in range(self.rng.randint(1, 3))] + + def gen_grants(self): + # One known-good grant for the securable. No valid privilege means no grants node: UC + # rejects a wrong one, and an empty one only reproduces the known drift bugs. + privilege = GRANT_PRIVILEGE.get(self.rtype) + if privilege is None: + return None + return [{"principal": DEFAULT_PRINCIPAL, "privileges": [privilege]}] + + def gen_permissions(self): + # As gen_grants: no valid level means no permissions node, rather than a random principal, a + # ${...} ref, or an empty list. + level = PERMISSION_LEVEL.get(self.rtype) + if level is None: + return None + return [{"level": level, "group_name": DEFAULT_PERMISSION_GROUP}] + + def gen_scalar(self, schema, name): + t = schema.get("type") + if t == "boolean": + # Cleanup must be able to destroy the bundle. + if name == "prevent_destroy": + return False + return self.rng.choice([True, False]) + if t == "integer": + # In hours, but UC accepts only a window of 0 or 7-30 days (0 or 168-720 hours). + if name == "custom_max_retention_hours": + return self.rng.choice([0, self.rng.randint(168, 720)]) + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_INTS) + return self.rng.choice([0, 1, self.rng.randint(2, 1000)]) + if t == "number": + return round(self.rng.uniform(0, 1000), 2) + # Fail loud on an unknown type; a missing type is "any" and falls through to string. + if t is not None and t not in SCALAR_TYPES: + sys.exit(f"gen_fuzz_config: unhandled schema type {t!r}") + # Pin cross-resource refs and typed-string fields to accepted values; a random token fails + # format/existence validation and drops the config. + if name == "catalog_name": + return DEFAULT_CATALOG + if name == "schema_name": + return DEFAULT_SCHEMA + if name == "warehouse_id": + return os.environ.get("TEST_DEFAULT_WAREHOUSE_ID", "") + if name == "notebook_path": + return NOTEBOOK_PATH + if name == "source_code_path": + return APP_SOURCE_CODE_PATH + if name == "parent_path": + return PARENT_PATH + if name == "file_path": + return FILE_PATH_BY_RESOURCE.get(self.rtype, self.token()) + if name.endswith("_duration") or name == "ttl": + return DURATION_VALUE + if name == "name" and self.rtype == "vector_search_indexes": + # UC requires the full catalog.schema.table name; each part is alphanumeric+_. + table = re.sub(r"[^0-9a-zA-Z_]", "_", f"fuzz_index_{self.unique}") + return f"{DEFAULT_CATALOG}.{DEFAULT_SCHEMA}.{table}" + if name in ("name", "display_name"): + return f"fuzz-{name}-{self.unique}" + # Free-form string with no pinned meaning (description, comment, tag): safe to probe + # dangerous input here, unlike the pinned fields above. + if self.rng.random() < DANGEROUS_PROB: + return self.rng.choice(DANGEROUS_STRINGS) + return self.token() + + def token(self): + return "fuzz_" + "".join(self.rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def resource_types(schema, gen): + # resources is oneOf[{ object with one property per resource type }]. + resources = gen.resolve(schema["properties"]["resources"]) + obj = next(b for b in resources["oneOf"] if b.get("type") == "object") + return obj["properties"] + + +def gen_resource(schema, gen, types, candidates, seed, unique): + rtype = gen.rng.choice(sorted(candidates)) + + # Each type is a map; the element schema is the object branch's additionalProperties. + map_schema = gen.resolve(types[rtype]) + obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") + element = obj["additionalProperties"] + + key = f"fuzz_{rtype}_{seed}" + gen.unique = unique + gen.rtype = rtype + instance = gen.gen(element, 0) + return rtype, key, instance + + +def gen_config(schema, seed, unique, allowed=frozenset()): + gen = Generator(schema, random.Random(seed), unique) + + types = resource_types(schema, gen) + candidates = [t for t in types if not allowed or t in allowed] + if not candidates: + sys.exit(f"no resource types to generate from (allowed={sorted(allowed)})") + + rtype, key, instance = gen_resource(schema, gen, types, candidates, seed, unique) + + return { + # Same name shape as the curated configs, so targets that derive workspace paths from the + # bundle name work unchanged. + "bundle": {"name": f"test-bundle-{unique}"}, + "resources": {rtype: {key: instance}}, + } + + +def to_yaml(obj, indent=0, list_item=False): + pad = " " * indent + if isinstance(obj, dict): + if not obj: + return f"{pad}{{}}\n" if not list_item else f"{pad}- {{}}\n" + out = "" + first = True + for k, v in obj.items(): + prefix = pad + "- " if list_item and first else (pad + " " if list_item else pad) + child_indent = indent + 2 if list_item else indent + 1 + if isinstance(v, (dict, list)) and v: + out += f"{prefix}{k}:\n" + to_yaml(v, child_indent) + else: + out += f"{prefix}{k}: {dump_scalar(v)}\n" + first = False + return out + if isinstance(obj, list): + if not obj: + return f"{pad}- []\n" if list_item else f"{pad}[]\n" + # A list inside a list: the marker needs its own line, else the two flatten into one. + if list_item: + return f"{pad}-\n" + to_yaml(obj, indent + 1) + out = "" + for item in obj: + if isinstance(item, (dict, list)): + out += to_yaml(item, indent, list_item=True) + else: + out += f"{pad}- {dump_scalar(item)}\n" + return out + return f"{pad}{dump_scalar(obj)}\n" + + +def dump_scalar(v): + # ensure_ascii=False keeps non-ASCII as literal UTF-8. The default escapes astral chars (e.g. + # the rocket probe) into surrogate pairs that YAML rejects, killing the config at parse time + # before it reaches bundle logic. Control chars stay escaped by json.dumps (YAML ok). + return json.dumps(v, ensure_ascii=False) diff --git a/acceptance/bin/gen_fuzz_config_check.py b/acceptance/bin/gen_fuzz_config_check.py new file mode 100755 index 00000000000..6e0db97b938 --- /dev/null +++ b/acceptance/bin/gen_fuzz_config_check.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +""" +Contract check for gen_fuzz_config.to_yaml: every scalar is on its own line as +`key: `, which mutate_fuzz_config's line-based loader relies on. Prints each case's +YAML (diffed by the harness) and exits non-zero on a violation. +""" + +import json +import os +import re +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import DANGEROUS_STRINGS, SKIP_PROPERTY_NAMES, gen_config, to_yaml + +# Tricky shapes: strings with ':' and '"', nested maps, lists of dicts, lists in lists, +# empty containers. +CASES = [ + {"comment": "value: with a colon", "description": 'quote " and : colon'}, + {"resources": {"jobs": {"j": {"name": "n", "tags": {"team": "jobs"}}}}}, + {"tasks": [{"description": "d", "timeout_seconds": 3600}, {"comment": "c"}]}, + {"nums": [0, 1, 2], "flag": True, "ratio": 1.5, "empty_map": {}, "empty_list": []}, + {"matrix": [[1, 2], [], {}]}, +] + +HEADER = re.compile(r"[\w.\-]+:$") # non-empty container: `key:` +SCALAR = re.compile(r"[\w.\-]+: (.+)$") # `key: ` + + +def check_line(line): + rest = line.lstrip(" ") + if rest == "-": + return # nested container marker; value is on following lines + rest = rest.removeprefix("- ") + if HEADER.fullmatch(rest): + return # container header; value is on following lines + m = SCALAR.fullmatch(rest) + if m: + json.loads(m.group(1)) # value must be single-line JSON + return + json.loads(rest) # bare list scalar: `- ` + + +def main(): + failed = False + for case in CASES: + text = to_yaml(case) + sys.stdout.write(text) + for line in text.splitlines(): + if not line.strip(): + continue + try: + check_line(line) + except ValueError: + sys.stderr.write(f"contract violation: not `key: `: {line!r}\n") + failed = True + + # Each DANGEROUS_STRINGS probe must still serialize to a single `key: ` line. + for i, val in enumerate(DANGEROUS_STRINGS): + line = to_yaml({"description": val}).rstrip("\n") + if "\n" in line or not SCALAR.fullmatch(line): + sys.stderr.write(f"DANGEROUS_STRINGS[{i}] broke the one-line scalar contract: {line!r}\n") + failed = True + + schema_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../bundle/schema/jsonschema.json") + with open(schema_path) as f: + schema = json.load(f) + seed24 = gen_config(schema, seed=24, unique="check", allowed={"registered_models"}) + rm = seed24["resources"]["registered_models"]["fuzz_registered_models_24"] + if SKIP_PROPERTY_NAMES & set(rm): + sys.stderr.write( + f"seed 24 registered_models emitted output-only fields: {sorted(SKIP_PROPERTY_NAMES & set(rm))}\n" + ) + failed = True + for field in ("name", "catalog_name", "schema_name"): + if field not in rm: + sys.stderr.write(f"seed 24 registered_models missing {field}\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/mutate_fuzz_config.py b/acceptance/bin/mutate_fuzz_config.py new file mode 100755 index 00000000000..41af5305b82 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config.py @@ -0,0 +1,258 @@ +""" +Mutate a known-good bundle config by deleting, perturbing, and adding random fields. + +Complements gen_fuzz_config.py: instead of building from the schema, this perturbs a curated +invariant config that already deploys, so it reaches a much higher deploy rate. + +Two mutation kinds, chosen per step: + +- destructive (always): delete a field or replace it with a token, a dangerous value, or an empty + container. Stays within the base's fields, so it finds only reject/panic bugs. +- additive (with a schema): inject a valid optional field the base omits, valued by the schema + generator. This is what reaches reconcile/drift bugs. + +The harness only asserts no-panic on fuzzed configs, so an invalid mutation is fine: the CLI must +reject it cleanly, not crash. + +Used as a library by emit_fuzz_config.py. +""" + +import os +import random +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from gen_fuzz_config import DANGEROUS_INTS, DANGEROUS_STRINGS, Generator, resource_types + +DANGEROUS = DANGEROUS_STRINGS + DANGEROUS_INTS + +# Chance a step injects a field rather than perturbing one. Biased high: injection is the path to +# drift bugs, and destructive coverage is already dense. +ADD_PROB = 0.6 + + +def tokenize(text): + # (indent, content) per non-blank, non-comment line. Only full-line comments are stripped; the + # curated bases have no trailing "#" in values. + out = [] + for raw in text.splitlines(): + stripped = raw.lstrip(" ") + if not stripped or stripped.startswith("#"): + continue + out.append((len(raw) - len(stripped), stripped.rstrip())) + return out + + +def scalar(text): + if text in ("", "null", "~"): + return None + # to_yaml emits empty containers in flow form; read them back so load -> emit -> load holds. + if text == "[]": + return [] + if text == "{}": + return {} + if text == "true": + return True + if text == "false": + return False + try: + return int(text) + except ValueError: + pass + try: + return float(text) + except ValueError: + pass + if len(text) >= 2 and text[0] == text[-1] and text[0] in "\"'": + return text[1:-1] + return text + + +def parse_block(tokens, i, indent): + if i >= len(tokens): + return {}, i + first = tokens[i][1] + if first.startswith("- ") or first == "-": + return parse_seq(tokens, i, indent) + if ": " in first or first.endswith(":"): + return parse_map(tokens, i, indent) + # Bare scalar: the whole block is a single value (e.g. a list scalar item). + return scalar(first), i + 1 + + +def parse_map(tokens, i, indent): + result = {} + while i < len(tokens) and tokens[i][0] == indent: + content = tokens[i][1] + if content.startswith("- "): + break + if ": " in content: + key, _, rest = content.partition(": ") + result[key.strip()] = scalar(rest) + i += 1 + elif content.endswith(":"): + key = content[:-1].strip() + i += 1 + if i < len(tokens) and tokens[i][0] > indent: + value, i = parse_block(tokens, i, tokens[i][0]) + else: + value = None + result[key] = value + else: + break + return result, i + + +def parse_seq(tokens, i, indent): + result = [] + while i < len(tokens) and tokens[i][0] == indent and (tokens[i][1].startswith("- ") or tokens[i][1] == "-"): + after = tokens[i][1][2:] if tokens[i][1].startswith("- ") else "" + child_indent = indent + 2 + # The item is its own block: the inline remainder (re-indented to child_indent) plus any + # deeper continuation lines that belong to it. + item = [] + if after: + item.append((child_indent, after)) + i += 1 + while i < len(tokens) and tokens[i][0] >= child_indent: + item.append(tokens[i]) + i += 1 + result.append(parse_block(item, 0, child_indent)[0] if item else None) + return result, i + + +def load_yaml(text): + tokens = tokenize(text) + value, _ = parse_block(tokens, 0, 0) + return value + + +def collect(node, out): + # (container, key) per child, so a mutation can delete or replace it in place. + if isinstance(node, dict): + for k, v in node.items(): + out.append((node, k)) + collect(v, out) + elif isinstance(node, list): + for idx, v in enumerate(node): + out.append((node, idx)) + collect(v, out) + + +def token(rng): + return "fuzz_" + "".join(rng.choice("abcdefghijklmnopqrstuvwxyz0123456789") for _ in range(8)) + + +def mutate_once(rng, roots): + refs = [] + for root in roots: + collect(root, refs) + if not refs: + return + container, key = rng.choice(refs) + op = rng.choice(["delete", "scalar", "dangerous", "empty"]) + if op == "delete": + del container[key] + elif op == "scalar": + container[key] = token(rng) + elif op == "dangerous": + container[key] = rng.choice(DANGEROUS) + else: + container[key] = rng.choice([{}, [], None]) + + +def resource_element(gen, type_schema): + # The instance schema is the map's object-branch additionalProperties. + map_schema = gen.resolve(type_schema) + obj = next(b for b in map_schema["oneOf"] if b.get("type") == "object") + return obj["additionalProperties"] + + +def collect_insertions(gen, node, schema, rtype, out): + # Record every writable optional field absent from an object, walking node and schema together + # so nested objects are candidates too. + schema = gen.resolve(schema) + if not isinstance(schema, dict): + return + + branches = schema.get("oneOf") or schema.get("anyOf") + if branches: + # Pick the branch matching the node we have, not a random one. + picked = None + for branch in branches: + resolved = gen.resolve(branch) + if isinstance(node, dict) and ( + resolved.get("type") == "object" or "properties" in resolved or gen.is_map(resolved) + ): + picked = resolved + break + if isinstance(node, list) and resolved.get("type") == "array": + picked = resolved + break + if picked is None: + return + schema = picked + + if isinstance(node, dict): + props = schema.get("properties", {}) + for name, prop_schema in props.items(): + if name not in node and not gen.should_skip_property(name, prop_schema): + out.append((node, name, prop_schema, rtype)) + for key, value in node.items(): + if key in props and isinstance(value, (dict, list)): + collect_insertions(gen, value, props[key], rtype, out) + if gen.is_map(schema): + for value in node.values(): + if isinstance(value, (dict, list)): + collect_insertions(gen, value, schema["additionalProperties"], rtype, out) + elif isinstance(node, list): + items = schema.get("items") + if items: + for value in node: + if isinstance(value, (dict, list)): + collect_insertions(gen, value, items, rtype, out) + + +def add_field(gen, rng, config): + # Inject one valid optional field, absent from the base, into a random insertion point. + types = resource_types(gen.root, gen) + points = [] + for rtype, instances in config.get("resources", {}).items(): + if rtype not in types or not isinstance(instances, dict): + continue + element = resource_element(gen, types[rtype]) + for instance in instances.values(): + if isinstance(instance, dict): + gen.rtype = rtype + collect_insertions(gen, instance, element, rtype, points) + if not points: + return + node, name, prop_schema, rtype = rng.choice(points) + # rtype drives grants/permissions/typed-string generation. + gen.rtype = rtype + value = gen.gen(prop_schema, 1, name) + if value is not None: + node[name] = value + + +def mutate(config, seed, schema=None, unique="fuzz"): + rng = random.Random(seed) + gen = Generator(schema, rng, unique) if schema is not None else None + + # Mutate only inside resource instances: keep the bundle/name and resources skeleton so there + # is always something to deploy, while every instance field is fair game. + roots = [] + for instances in config.get("resources", {}).values(): + if isinstance(instances, dict): + roots.extend(v for v in instances.values() if isinstance(v, (dict, list))) + + for _ in range(rng.randint(1, 3)): + # gen is None short-circuits before rng is touched, so the no-schema path keeps its exact + # RNG stream unchanged. + if gen is not None and rng.random() < ADD_PROB: + add_field(gen, rng, config) + else: + mutate_once(rng, roots) + + return config diff --git a/acceptance/bin/mutate_fuzz_config_check.py b/acceptance/bin/mutate_fuzz_config_check.py new file mode 100755 index 00000000000..28c89cab173 --- /dev/null +++ b/acceptance/bin/mutate_fuzz_config_check.py @@ -0,0 +1,102 @@ +#!/usr/bin/env python3 +""" +Contract check for mutate_fuzz_config's minimal YAML loader and mutation engine +(the harness diffs stdout; a non-zero exit marks a violation on stderr): + +- The loader round-trips every curated base: load -> to_yaml -> load is a fixed point, so + a base the loader can't represent is caught here, not as a confusing fuzz failure. +- Mutation is deterministic for a fixed seed (reproducible repros). + +It also prints a few mutated configs so an algorithm change shows up as an output diff. +""" + +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + +from emit_fuzz_config import MUTATE_BASES +from envsubst import substitute_variables +from gen_fuzz_config import SKIP_PROPERTY_NAMES, to_yaml +from mutate_fuzz_config import load_yaml, mutate + +CONFIGS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "bundle", "invariant", "configs") +SCHEMA = os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "bundle", "schema", "jsonschema.json") + + +def render(name): + with open(os.path.join(CONFIGS, name + ".yml.tmpl")) as f: + return substitute_variables(f.read()) + + +def instance(config): + # The curated bases are single-resource; return that one resource instance. + (instances,) = config["resources"].values() + (value,) = instances.values() + return value + + +def main(): + # Fixed so the printed configs are stable regardless of the harness's unique name. + os.environ["UNIQUE_NAME"] = "check" + failed = False + + for name in MUTATE_BASES: + text = render(name) + parsed = load_yaml(text) + if not isinstance(parsed, dict) or "resources" not in parsed: + sys.stderr.write(f"{name}: base did not parse to a config with resources\n") + failed = True + continue + # load -> emit -> load is a fixed point. + if load_yaml(to_yaml(parsed)) != parsed: + sys.stderr.write(f"{name}: loader is not a round-trip fixed point\n") + failed = True + + # Mutation must be reproducible for a fixed seed. + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("volume")), seed)) + b = to_yaml(mutate(load_yaml(render("volume")), seed)) + if a != b: + sys.stderr.write(f"seed {seed}: mutation is not deterministic\n") + failed = True + + for seed in range(3): + sys.stdout.write(f"=== volume seed={seed} ===\n") + sys.stdout.write(to_yaml(mutate(load_yaml(render("volume")), seed))) + + # Assert-only (no stdout) so this doesn't churn as the schema grows. The registered_model + # base sets no optional fields, so any added field must have been injected. + with open(SCHEMA) as f: + schema = json.load(f) + + for seed in range(5): + a = to_yaml(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check")) + b = to_yaml(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check")) + if a != b: + sys.stderr.write(f"seed {seed}: schema-aware mutation is not deterministic\n") + failed = True + + base_fields = set(instance(load_yaml(render("registered_model")))) + injected = False + for seed in range(30): + fields = set(instance(mutate(load_yaml(render("registered_model")), seed, schema=schema, unique="check"))) + added = fields - base_fields + if added: + injected = True + # Injecting an output-only field would manufacture false drift. + leaked = SKIP_PROPERTY_NAMES & added + if leaked: + sys.stderr.write(f"seed {seed}: injected output-only field(s): {sorted(leaked)}\n") + failed = True + if not injected: + sys.stderr.write("schema-aware mutation never injected an optional field\n") + failed = True + + if failed: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/acceptance/bin/verify_no_drift.py b/acceptance/bin/verify_no_drift.py index 9b272c1ce79..4d4a3033776 100755 --- a/acceptance/bin/verify_no_drift.py +++ b/acceptance/bin/verify_no_drift.py @@ -11,6 +11,10 @@ def check_plan(path): with open(path) as fobj: raw = fobj.read() + # A failed `bundle plan` leaves nothing to check; say so instead of raising below. + if not raw.strip(): + sys.exit(f"{path}: empty plan output (bundle plan failed)") + changes_detected = 0 try: diff --git a/acceptance/bundle/invariant/README.md b/acceptance/bundle/invariant/README.md index 184d3f541c4..9e86a1db4b5 100644 --- a/acceptance/bundle/invariant/README.md +++ b/acceptance/bundle/invariant/README.md @@ -4,3 +4,28 @@ no_drift test checks that there are no actions planned after successful deploy. test will dump full JSON plan to the output. In order to add a new test, add a config to configs/ and include it in test.toml. + +The fuzz/ test runs generated configs through a real invariant test script (see fuzz/script). +Both the target and the way configs are built are matrixed in fuzz/test.toml: + +`FUZZ_TARGET` picks the invariant, and each one is also a curated invariant test that runs +over the `INPUT_CONFIG` matrix: + +- `no_drift` -- deploy, then no drift +- `migrate` -- Terraform deploy, migrate to direct, then no drift +- `delete_idempotent` -- deploy, delete by emptying the config, then re-run the delete on restored state +- `destroy_idempotent` -- deploy, destroy, then destroy again on restored state + +`FUZZ_MODE` picks how the config is built: + +- `generate` -- build a random resource by walking the live `databricks bundle schema` +- `mutate` -- perturb one of the curated configs (see MUTATE_BASES in emit_fuzz_config.py) + +Free-form scalars are occasionally replaced with dangerous / near-range-end values (empty, +whitespace, over-long, control characters, int32/int64 boundaries) to probe the CLI's input +handling. + +Since the schema comes from the CLI under test, an unrelated struct change can shift a +seed onto a new config. A failure is a real CLI bug (panic, internal error, or drift), +not flakiness; the failing seed's `LOG.repro` prints a ready-to-run repro, of the form +`FUZZ_SEED_START= FUZZ_SEED_COUNT=1 FUZZ_TARGET=no_drift FUZZ_MODE=generate task test-fuzz`. diff --git a/acceptance/bundle/invariant/fuzz/out.test.toml b/acceptance/bundle/invariant/fuzz/out.test.toml new file mode 100644 index 00000000000..8390c52e534 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/out.test.toml @@ -0,0 +1,12 @@ +Local = true +Cloud = false +RequiresUnityCatalog = true +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["direct"] +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] +EnvMatrix.FUZZ_TARGET = [ + "no_drift", + "migrate", + "delete_idempotent", + "destroy_idempotent" +] +EnvMatrix.INPUT_CONFIG = [] diff --git a/acceptance/bundle/invariant/fuzz/output.txt b/acceptance/bundle/invariant/fuzz/output.txt new file mode 100644 index 00000000000..e69de29bb2d diff --git a/acceptance/bundle/invariant/fuzz/script b/acceptance/bundle/invariant/fuzz/script new file mode 100644 index 00000000000..8515a30218d --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/script @@ -0,0 +1,137 @@ +# Invariant fuzzing: generate a random config per seed and run the real invariant script, which +# reaches the generator through the helper overrides in script.prepare. Those scripts print +# INPUT_CONFIG_OK once a config deploys, so a non-zero result before the marker is a rejection; +# a panic anywhere, or a failure after it, is a bug. + +START="${FUZZ_SEED_START:-0}" +COUNT="${FUZZ_SEED_COUNT:-5}" + +# Per-seed cap: a seed past this budget is stuck, not slow. Needs GNU timeout; else uncapped. +SEED_TIMEOUT="${FUZZ_SEED_TIMEOUT:-180}" + +# Overall budget (seconds): stop starting new seeds past it and exit cleanly, so a slow-but- +# progressing variant isn't force-killed at the per-script Timeout and read as a failure. +# 900s leaves margin under the 20m test.toml Timeout. Set FUZZ_TIME_BUDGET=0 to disable. +BUDGET="${FUZZ_TIME_BUDGET:-900}" + +# no_drift/script reads READPLAN via readplanarg; the fuzzer skips the saved-plan matrix. +export READPLAN="" + +# Emit the schema from the CLI under test so the generator always matches it. +$CLI bundle schema > schema.json 2>LOG.schema.err +cat LOG.schema.err | contains.py '!panic:' '!internal error' > /dev/null + +# Factored out so it can run directly or under `timeout`. +seed_body() { + cd "$1" + # Seeds share one long-lived workspace, so scope the unique name to the seed; otherwise state + # a seed leaves behind reads back as drift in the next one. Scoped here rather than in the + # generator so targets that derive workspace paths from $UNIQUE_NAME see it too. + export UNIQUE_NAME="$UNIQUE_NAME-$2" + export FUZZ_SEED="$2" + export FUZZ_SCHEMA="../schema.json" + source "$TESTDIR/../${FUZZ_TARGET:-no_drift}/script" +} + +# timeout spawns a fresh bash, so export seed_body and the harness helpers it calls (trace). +export -f $(compgen -A function) + +run_seed() { + local dir="$1" seed="$2" + # SEED_TIMEOUT=0 or no timeout binary: run uncapped (matches the hang reproduce hint). + if [ "$SEED_TIMEOUT" != "0" ] && command -v timeout > /dev/null 2>&1; then + # SIGQUIT first for Go's goroutine dump, then SIGKILL as a backstop. + timeout --signal=QUIT --kill-after=10s "$SEED_TIMEOUT" \ + bash -euo pipefail -c 'seed_body "$@"' _ "$dir" "$seed" > "$dir/LOG.check" 2>&1 + else + # set -euo pipefail explicitly: the caller disables errexit to capture the seed's exit + # code and a plain subshell inherits that. Without errexit a rejected config runs on to + # INPUT_CONFIG_OK and reads as a bug. + ( set -euo pipefail; seed_body "$dir" "$seed" ) > "$dir/LOG.check" 2>&1 + fi +} + +# One machine-readable line per seed. To a file, not stdout, so the committed run's empty-output +# assertion holds. +record() { + echo "$1 seed=$2 target=${FUZZ_TARGET:-no_drift} mode=${FUZZ_MODE:-generate}" >> LOG.summary +} + +# The repro goes to a file because the harness rewrites env-var values in stdout. +fail() { + local seed="$1" kind="$2" reason="$3" prefix="${4:-}" + record "$kind" "$seed" + echo "fuzz: seed $seed $reason, reproduce with: ${prefix}FUZZ_SEED_START=$seed FUZZ_SEED_COUNT=1 FUZZ_TARGET=${FUZZ_TARGET:-no_drift} FUZZ_MODE=${FUZZ_MODE:-generate} task test-fuzz" > LOG.repro + exit 1 +} + +for ((offset = 0; offset < COUNT; offset++)); do + # A clean stop, not a failure, so log to a file. + if [ "$BUDGET" != "0" ] && [ "$SECONDS" -ge "$BUDGET" ]; then + echo "fuzz: stopping after $offset/$COUNT seeds; hit FUZZ_TIME_BUDGET=${BUDGET}s" > LOG.budget + break + fi + + seed=$((START + offset)) + dir="seed-$seed" + mkdir -p "$dir" + + set +e + run_seed "$dir" "$seed" + rc=$? + set -e + + if [ "$rc" -eq 0 ]; then + record deployed "$seed" + continue + fi + + # timeout exits 124 (137 if the SIGKILL backstop fired): the seed hung, which is distinct + # from a drift bug. + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + fail "$seed" hang "hung (>${SEED_TIMEOUT}s)" "FUZZ_SEED_TIMEOUT=0 " + fi + + # The generator only writes to stderr when it fails: our bug, not a rejected config. + if [ -s "$dir/LOG.gen.err" ]; then + fail "$seed" bug "could not be generated: $(head -1 "$dir/LOG.gen.err")" + fi + + # A panic or internal error anywhere is a bug even if the CLI then rejects the config. + if ! cat "$dir"/LOG.* 2>/dev/null | contains.py '!panic:' '!internal error' > /dev/null; then + fail "$seed" bug "panicked or hit an internal error" + fi + + # A 501 is a testserver gap. Checked before the marker below, else an unstubbed route during + # plan or destroy reads as a drift bug. + if grep -qs "No stub found for pattern" "$dir"/LOG.*; then + record gap "$seed" + continue + fi + + # Failing after INPUT_CONFIG_OK means the config deployed but drifted (or destroy failed); + # failing before it with no panic just means the config was rejected. + if grep -q INPUT_CONFIG_OK "$dir/LOG.check"; then + fail "$seed" bug "broke the invariant" + fi + + record rejected "$seed" +done + +# Per-variant tally for triage. Reached only on a clean run; a bug/hang exits above. +if [ -f LOG.summary ]; then + # Snapshot the counts before appending the header, else awk would also count it. + totals=$(awk '{print $1}' LOG.summary | sort | uniq -c) + { + echo "--- totals ---" + echo "$totals" + } >> LOG.summary +fi + +# Nothing deploying is not a pass: it means the schema, generator or fixtures are broken, which +# otherwise looks just like the CLI correctly rejecting random input. A single-seed replay is +# exempt, where one rejected config is a normal outcome. +if [ "$COUNT" -gt 1 ] && ! grep -qs '^deployed ' LOG.summary; then + echo "fuzz: no seed deployed; the schema, generator or fixtures are broken" >&2 + exit 1 +fi diff --git a/acceptance/bundle/invariant/fuzz/script.prepare b/acceptance/bundle/invariant/fuzz/script.prepare new file mode 100644 index 00000000000..047204ca1a0 --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/script.prepare @@ -0,0 +1,40 @@ +# Fuzz overrides of the shared invariant helpers, which the harness concatenates from +# ../script.prepare before this file. The target scripts stay unaware of fuzzing. + +# The config comes from the generator rather than configs/$INPUT_CONFIG, which the fuzzer leaves +# unset. The validate probe lives here because the config only exists once it is rendered and +# must be probed before the target's deploy. Its output is redirected, not recorded, as a fuzzed +# config may warn. +invariant_render() { + # Stage the fixtures the generator's file_path/source_code_path fields point at. + cp -r "$TESTDIR/../data/." . &> LOG.cp + + emit_fuzz_config.py > databricks.yml 2>LOG.gen.err + cp databricks.yml LOG.config + + trace $CLI bundle validate &> LOG.validate + cat LOG.validate | contains.py '!panic:' '!internal error' > /dev/null +} + +# A generated config can deploy and still legitimately differ from the fake server, which does +# not round-trip every field, so the exact check false-positives. Swap in the one oracle that +# does not depend on server fidelity: planning is deterministic, so two consecutive plans of the +# same state must be byte-identical. Nightly runs set FUZZ_CHECK_DRIFT to keep the exact check. +if [ -z "${FUZZ_CHECK_DRIFT:-}" ]; then + invariant_verify_no_drift() { + # Compare only when both plans succeed -- a plan that fails on an unstubbed read is a gap + # and can differ run to run. + set +e + $CLI bundle plan -o json > LOG.plan1.json 2>LOG.plan1.err + local plan1_rc=$? + $CLI bundle plan -o json > LOG.plan2.json 2>LOG.plan2.err + local plan2_rc=$? + set -e + cat LOG.plan1.err | contains.py '!panic:' '!internal error' > /dev/null + cat LOG.plan2.err | contains.py '!panic:' '!internal error' > /dev/null + if [ "$plan1_rc" -eq 0 ] && [ "$plan2_rc" -eq 0 ]; then + # diff exits non-zero on any difference; under set -e that fails the seed as a bug. + diff LOG.plan1.json LOG.plan2.json > LOG.plan.determinism.diff + fi + } +fi diff --git a/acceptance/bundle/invariant/fuzz/test.toml b/acceptance/bundle/invariant/fuzz/test.toml new file mode 100644 index 00000000000..77d2d27168f --- /dev/null +++ b/acceptance/bundle/invariant/fuzz/test.toml @@ -0,0 +1,31 @@ +# The fuzzer generates its own configs, so drop the inherited INPUT_CONFIG matrix. +EnvMatrix.INPUT_CONFIG = [] + +# Local only: against a real workspace each seed's deploy/migrate/plan/destroy round trip +# takes minutes and trips SEED_TIMEOUT, and the cloud run leaves FUZZ_CHECK_DRIFT unset, so it +# would only re-assert the no-panic property the local run already covers. +Cloud = false + +# Room for the nightly FUZZ_TIME_BUDGET (script) plus the last seed's tail. +Timeout = '20m' + +# A resource type the testserver doesn't model is a coverage gap, not a failure: 501 lets the +# script record the seed as a gap instead of failing the whole run. +IgnoreUnhandledRequests = true + +# The idempotency targets assert a delete or destroy re-run succeeds, which holds regardless of +# how faithfully the fake server round-trips fields. No redeploy target: no_drift's post-deploy +# plan already dry-runs one. +EnvMatrix.FUZZ_TARGET = ["no_drift", "migrate", "delete_idempotent", "destroy_idempotent"] + +# Snapshot of pre-delete state the idempotency targets keep; may linger if a seed fails. +Ignore = [".databricks.backup"] + +# generate = build from the schema; mutate = perturb a curated config. See emit_fuzz_config.py. +EnvMatrix.FUZZ_MODE = ["generate", "mutate"] + +# Generate mode pins name/display_name, so it cannot vary the identifier the delete path reads, +# and ../delete_idempotent already covers every resource type. Mutate can hit the identifier and +# reshape grants/permissions, so it is the mode worth running here. +EnvMatrixExclude.no_generate_on_delete_idempotent = ["FUZZ_MODE=generate", "FUZZ_TARGET=delete_idempotent"] +EnvMatrixExclude.no_generate_on_destroy_idempotent = ["FUZZ_MODE=generate", "FUZZ_TARGET=destroy_idempotent"] diff --git a/acceptance/bundle/invariant/migrate/script b/acceptance/bundle/invariant/migrate/script index 4cd19571951..a3829ed039f 100644 --- a/acceptance/bundle/invariant/migrate/script +++ b/acceptance/bundle/invariant/migrate/script @@ -13,7 +13,7 @@ MIGRATE_ARGS="" # (order-sensitive), terraform plan reports positional drift when the bundle config # specifies depends_on in a different order than the provider's sorted state. # This is a false positive -- the logical dependencies are identical. -if [[ "$INPUT_CONFIG" == "job_with_depends_on.yml.tmpl" ]]; then +if [[ "${INPUT_CONFIG:-}" == "job_with_depends_on.yml.tmpl" ]]; then MIGRATE_ARGS="--noplancheck" fi diff --git a/acceptance/bundle/invariant/script.prepare b/acceptance/bundle/invariant/script.prepare index 54f2a3dbf25..26f6caab62c 100644 --- a/acceptance/bundle/invariant/script.prepare +++ b/acceptance/bundle/invariant/script.prepare @@ -4,7 +4,8 @@ invariant_cleanup() { trace $CLI bundle destroy --auto-approve &> LOG.destroy cat LOG.destroy | contains.py '!panic:' '!internal error' > /dev/null - CLEANUP_SCRIPT="$TESTDIR/../configs/$INPUT_CONFIG-cleanup.sh" + # INPUT_CONFIG is unset for a caller that generates its own config, and scripts run under set -u. + CLEANUP_SCRIPT="$TESTDIR/../configs/${INPUT_CONFIG:-}-cleanup.sh" if [ -f "$CLEANUP_SCRIPT" ]; then source "$CLEANUP_SCRIPT" &> LOG.cleanup fi diff --git a/acceptance/internal/config.go b/acceptance/internal/config.go index f981e790e0b..adc0bec21d8 100644 --- a/acceptance/internal/config.go +++ b/acceptance/internal/config.go @@ -83,6 +83,10 @@ type TestConfig struct { // out.requests.txt RecordRequests *bool + // Return 501 for a request with no handler instead of failing the test: a resource type the + // testserver doesn't model is a coverage gap, not a bug. + IgnoreUnhandledRequests *bool + // List of request headers to include when recording requests. IncludeRequestHeaders []string diff --git a/acceptance/internal/prepare_server.go b/acceptance/internal/prepare_server.go index 3b1be7a79b3..5c1c6591631 100644 --- a/acceptance/internal/prepare_server.go +++ b/acceptance/internal/prepare_server.go @@ -151,7 +151,7 @@ func PrepareServerAndClient(t *testing.T, config TestConfig, logRequests bool, o // Default case. Start a dedicated local server for the test with the server stubs configured // as overrides. - host := startLocalServer(t, config.Server, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir) + host := startLocalServer(t, config.Server, recordRequests, logRequests, config.IncludeRequestHeaders, outputDir, isTruePtr(config.IgnoreUnhandledRequests)) cfg := &sdkconfig.Config{ Host: host, Token: token, @@ -204,8 +204,10 @@ func startLocalServer(t *testing.T, logRequests bool, includeHeaders []string, outputDir string, + ignoreUnhandledRequests bool, ) string { s := testserver.New(t) + s.IgnoreUnhandledRequests = ignoreUnhandledRequests // Record API requests in out.requests.txt if RecordRequests is true // in test.toml diff --git a/acceptance/selftest/gen_fuzz_config/out.test.toml b/acceptance/selftest/gen_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/gen_fuzz_config/output.txt b/acceptance/selftest/gen_fuzz_config/output.txt new file mode 100644 index 00000000000..ec780260c85 --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/output.txt @@ -0,0 +1,26 @@ +comment: "value: with a colon" +description: "quote \" and : colon" +resources: + jobs: + j: + name: "n" + tags: + team: "jobs" +tasks: + - description: "d" + timeout_seconds: 3600 + - comment: "c" +nums: + - 0 + - 1 + - 2 +flag: true +ratio: 1.5 +empty_map: {} +empty_list: [] +matrix: + - + - 1 + - 2 + - [] + - {} diff --git a/acceptance/selftest/gen_fuzz_config/script b/acceptance/selftest/gen_fuzz_config/script new file mode 100644 index 00000000000..2737c67674d --- /dev/null +++ b/acceptance/selftest/gen_fuzz_config/script @@ -0,0 +1 @@ +gen_fuzz_config_check.py diff --git a/acceptance/selftest/mutate_fuzz_config/out.test.toml b/acceptance/selftest/mutate_fuzz_config/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/selftest/mutate_fuzz_config/output.txt b/acceptance/selftest/mutate_fuzz_config/output.txt new file mode 100644 index 00000000000..4b7289cfa51 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/output.txt @@ -0,0 +1,36 @@ +=== volume seed=0 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: "main" + schema_name: [] + grants: + - principal: "account users" +=== volume seed=1 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + catalog_name: " " + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" +=== volume seed=2 === +bundle: + name: "test-bundle-check" +resources: + volumes: + foo: + name: "test-volume-check" + schema_name: "default" + grants: + - principal: "account users" + privileges: + - "READ_VOLUME" diff --git a/acceptance/selftest/mutate_fuzz_config/script b/acceptance/selftest/mutate_fuzz_config/script new file mode 100644 index 00000000000..2d57c739261 --- /dev/null +++ b/acceptance/selftest/mutate_fuzz_config/script @@ -0,0 +1 @@ +mutate_fuzz_config_check.py diff --git a/libs/testserver/catalogs.go b/libs/testserver/catalogs.go index 351c5a76499..c34537ff776 100644 --- a/libs/testserver/catalogs.go +++ b/libs/testserver/catalogs.go @@ -24,21 +24,38 @@ func (s *FakeWorkspace) CatalogsCreate(req Request) Response { } } + // An empty name would store a catalog the CLI cannot find again. UC rejects it; verified + // against a real workspace. + if createRequest.Name == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": `Invalid input: RPC CreateCatalog Field managedcatalog.CatalogInfo.name: name "" is not a valid name. Valid names cannot contain spaces, periods, forward slashes, or control characters.`, + }, + } + } + + // Echo back every field create accepts: a dropped one makes the next plan see a + // phantom change. catalogInfo := catalog.CatalogInfo{ - Name: createRequest.Name, - Comment: createRequest.Comment, - StorageRoot: createRequest.StorageRoot, - ProviderName: createRequest.ProviderName, - ShareName: createRequest.ShareName, - Options: createRequest.Options, - Properties: createRequest.Properties, - FullName: createRequest.Name, - CreatedAt: nowMilli(), - CreatedBy: s.CurrentUser().UserName, - UpdatedBy: s.CurrentUser().UserName, - MetastoreId: nextUUID(), - Owner: s.CurrentUser().UserName, - CatalogType: catalog.CatalogTypeManagedCatalog, + Name: createRequest.Name, + Comment: createRequest.Comment, + ConnectionName: createRequest.ConnectionName, + CustomMaxRetentionHours: createRequest.CustomMaxRetentionHours, + ManagedEncryptionSettings: createRequest.ManagedEncryptionSettings, + StorageRoot: createRequest.StorageRoot, + ProviderName: createRequest.ProviderName, + ShareName: createRequest.ShareName, + Options: createRequest.Options, + Properties: createRequest.Properties, + FullName: createRequest.Name, + CreatedAt: nowMilli(), + CreatedBy: s.CurrentUser().UserName, + UpdatedBy: s.CurrentUser().UserName, + MetastoreId: nextUUID(), + Owner: s.CurrentUser().UserName, + CatalogType: catalog.CatalogTypeManagedCatalog, } catalogInfo.UpdatedAt = catalogInfo.CreatedAt if catalogInfo.Properties == nil && createRequest.Name == catalogNameManagedDefaults { diff --git a/libs/testserver/models.go b/libs/testserver/models.go index febfd8bf8a8..4f7cebcd79b 100644 --- a/libs/testserver/models.go +++ b/libs/testserver/models.go @@ -19,6 +19,18 @@ func (s *FakeWorkspace) ModelRegistryCreateModel(req Request) any { } } + // An empty name would store a model the CLI cannot find again. MLflow rejects it; verified + // against a real workspace. + if request.Name == "" { + return Response{ + StatusCode: 400, + Body: map[string]string{ + "error_code": "INVALID_PARAMETER_VALUE", + "message": "Got an invalid name ''. Registered Model names cannot be empty strings.", + }, + } + } + // Create the model with a numeric ID (matching real API behavior) modelId := strconv.FormatInt(nextID(), 10) model := ml.Model{ diff --git a/libs/testserver/server.go b/libs/testserver/server.go index 5ae3141bfd2..d6d82da7042 100644 --- a/libs/testserver/server.go +++ b/libs/testserver/server.go @@ -73,6 +73,10 @@ type Server struct { RequestCallback func(request *Request) ResponseCallback func(request *Request, response *EncodedResponse) + + // Return 501 for a request with no handler instead of failing the test: the fuzzer emits + // resource types the testserver may not model. Curated tests leave it false so gaps stay loud. + IgnoreUnhandledRequests bool } type Request struct { @@ -278,7 +282,11 @@ func New(t testutil.TestingT) *Server { body = fmt.Sprintf("[%d bytes] %s", len(bodyBytes), bodyBytes) } - t.Errorf(`No handler for URL: %s + if s.IgnoreUnhandledRequests { + // Coverage gap, not a CLI bug: log it but return the 501 so the caller can reject. + t.Logf("No handler for URL (ignored): %s", r.URL) + } else { + t.Errorf(`No handler for URL: %s Body: %s For acceptance tests, add this to test.toml: @@ -287,6 +295,7 @@ Pattern = %q Response.Body = '' # Response.StatusCode = `, r.URL, body, pattern) + } w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusNotImplemented) diff --git a/libs/testserver/server_test.go b/libs/testserver/server_test.go index 6c6dfe8c160..141a873c9bd 100644 --- a/libs/testserver/server_test.go +++ b/libs/testserver/server_test.go @@ -3,12 +3,54 @@ package testserver_test import ( "net/http" "net/http/httptest" + "sync" "testing" + "github.com/databricks/cli/internal/testutil" "github.com/databricks/cli/libs/testserver" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +// recordingT wraps a real TestingT but records Errorf calls instead of failing, +// so a test can assert whether the server would have failed the run. +type recordingT struct { + testutil.TestingT + mu sync.Mutex + errCount int +} + +func (r *recordingT) Errorf(format string, args ...any) { + r.mu.Lock() + defer r.mu.Unlock() + r.errCount++ +} + +func (r *recordingT) errors() int { + r.mu.Lock() + defer r.mu.Unlock() + return r.errCount +} + +func TestIgnoreUnhandledRequests(t *testing.T) { + for _, ignore := range []bool{false, true} { + rt := &recordingT{TestingT: t} + s := testserver.New(rt) + s.IgnoreUnhandledRequests = ignore + + resp, err := http.Get(s.URL + "/api/2.0/no-such-endpoint") + require.NoError(t, err) + assert.Equal(t, http.StatusNotImplemented, resp.StatusCode) + require.NoError(t, resp.Body.Close()) + + if ignore { + assert.Zero(t, rt.errors(), "unhandled request must not fail the test when ignored") + } else { + assert.Positive(t, rt.errors(), "unhandled request must fail the test by default") + } + } +} + func TestIsLocalhostProbe(t *testing.T) { tests := []struct { name string