diff --git a/acceptance/experimental/air/convert-to-dabs/docker.yaml b/acceptance/experimental/air/convert-to-dabs/docker.yaml new file mode 100644 index 00000000000..848ac77fd9c --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/docker.yaml @@ -0,0 +1,8 @@ +experiment_name: docker-test +command: python train.py +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +environment: + docker_image: + url: myregistry/img:tag diff --git a/acceptance/experimental/air/convert-to-dabs/out.test.toml b/acceptance/experimental/air/convert-to-dabs/out.test.toml new file mode 100644 index 00000000000..f784a183258 --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/out.test.toml @@ -0,0 +1,3 @@ +Local = true +Cloud = false +EnvMatrix.DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] diff --git a/acceptance/experimental/air/convert-to-dabs/output.txt b/acceptance/experimental/air/convert-to-dabs/output.txt new file mode 100644 index 00000000000..45c1b476bd9 --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/output.txt @@ -0,0 +1,107 @@ + +=== convert an AIR run YAML into a DABs bundle (in place, next to the source) +>>> [CLI] experimental air convert-to-dabs train.yaml +Wrote a Databricks Asset Bundle to .: + databricks.yml + training_config.yaml + command.sh + +Notes: + - code_source points at your source directory; bundle deploy packages and uploads it from there. + +To deploy and run this workload as a bundle: + 1. cd . + 2. [CLI] bundle validate + 3. [CLI] bundle deploy + 4. [CLI] bundle run torchrun-a10-smoke-test + +bundle deploy uploads the code source and launch scripts automatically. + +Run these with this same CLI: a build without ai_runtime_task support +only warns about the unknown field, then deploys a job with no AI task. + +Unlike `air run` (which submits an ephemeral run), bundle deploy creates a +persistent job that is not garbage-collected. When you are done, remove the +job and its uploaded files with: + [CLI] bundle destroy + +=== emitted databricks.yml (code_source_path points at ./src; no code is copied) +>>> cat databricks.yml +bundle: + name: torchrun-a10-smoke-test +targets: + dev: + mode: development + default: true +resources: + jobs: + torchrun-a10-smoke-test: + name: torchrun-a10-smoke-test + tasks: + - task_key: torchrun-a10-smoke-test + environment_key: default + ai_runtime_task: + experiment: torchrun-a10-smoke-test + deployments: + - command_path: ./command.sh + compute: + accelerator_type: GPU_1xA10 + accelerator_count: 1 + code_source_path: ./src + environments: + - environment_key: default + spec: + environment_version: "5" + dependencies: + - numpy + +=== the generated command.sh carries the run command +>>> cat command.sh +torchrun --nproc_per_node=1 train.py +=== the emitted bundle validates +>>> [CLI] bundle validate +Name: torchrun-a10-smoke-test +Target: dev +Workspace: + User: [USERNAME] + Path: /Workspace/Users/[USERNAME]/.bundle/torchrun-a10-smoke-test/dev + +Validation OK! + +=== re-converting refuses to clobber the generated bundle +>>> [CLI] experimental air convert-to-dabs train.yaml +Error: databricks.yml already exists in .; pass --force to overwrite or remove it + +Exit code: 1 + +=== --force overwrites it +>>> [CLI] experimental air convert-to-dabs train.yaml --force +Wrote a Databricks Asset Bundle to .: + databricks.yml + training_config.yaml + command.sh + +Notes: + - code_source points at your source directory; bundle deploy packages and uploads it from there. + +To deploy and run this workload as a bundle: + 1. cd . + 2. [CLI] bundle validate + 3. [CLI] bundle deploy + 4. [CLI] bundle run torchrun-a10-smoke-test + +bundle deploy uploads the code source and launch scripts automatically. + +Run these with this same CLI: a build without ai_runtime_task support +only warns about the unknown field, then deploys a job with no AI task. + +Unlike `air run` (which submits an ephemeral run), bundle deploy creates a +persistent job that is not garbage-collected. When you are done, remove the +job and its uploaded files with: + [CLI] bundle destroy + +=== docker_image is not supported yet +>>> [CLI] experimental air convert-to-dabs docker.yaml --output-dir generated-docker +Error: environment.docker_image is not yet supported by convert-to-dabs + +Exit code: 1 diff --git a/acceptance/experimental/air/convert-to-dabs/script b/acceptance/experimental/air/convert-to-dabs/script new file mode 100644 index 00000000000..8882963ac91 --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/script @@ -0,0 +1,20 @@ +title "convert an AIR run YAML into a DABs bundle (in place, next to the source)" +trace $CLI experimental air convert-to-dabs train.yaml + +title "emitted databricks.yml (code_source_path points at ./src; no code is copied)" +trace cat databricks.yml + +title "the generated command.sh carries the run command" +trace cat command.sh + +title "the emitted bundle validates" +trace $CLI bundle validate + +title "re-converting refuses to clobber the generated bundle" +errcode trace $CLI experimental air convert-to-dabs train.yaml + +title "--force overwrites it" +trace $CLI experimental air convert-to-dabs train.yaml --force + +title "docker_image is not supported yet" +errcode trace $CLI experimental air convert-to-dabs docker.yaml --output-dir generated-docker diff --git a/acceptance/experimental/air/convert-to-dabs/src/train.py b/acceptance/experimental/air/convert-to-dabs/src/train.py new file mode 100644 index 00000000000..c859094afdf --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/src/train.py @@ -0,0 +1 @@ +print("train") diff --git a/acceptance/experimental/air/convert-to-dabs/test.toml b/acceptance/experimental/air/convert-to-dabs/test.toml new file mode 100644 index 00000000000..386f44a60f0 --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/test.toml @@ -0,0 +1,3 @@ +# convert-to-dabs writes the bundle in place (next to train.yaml + src/). These are +# generated artifacts, not committed inputs, so exclude them from the repo-diff check. +Ignore = ["databricks.yml", "command.sh", "training_config.yaml", "generated-docker"] diff --git a/acceptance/experimental/air/convert-to-dabs/train.yaml b/acceptance/experimental/air/convert-to-dabs/train.yaml new file mode 100644 index 00000000000..bae47bc4dbd --- /dev/null +++ b/acceptance/experimental/air/convert-to-dabs/train.yaml @@ -0,0 +1,13 @@ +experiment_name: torchrun-a10-smoke-test +command: torchrun --nproc_per_node=1 train.py +compute: + accelerator_type: GPU_1xA10 + num_accelerators: 1 +environment: + version: 5 + dependencies: + - numpy +code_source: + type: snapshot + snapshot: + root_path: ./src diff --git a/acceptance/experimental/air/help/output.txt b/acceptance/experimental/air/help/output.txt index ee89e778d6b..5cc28fd3628 100644 --- a/acceptance/experimental/air/help/output.txt +++ b/acceptance/experimental/air/help/output.txt @@ -10,12 +10,13 @@ Usage: databricks experimental air [command] Available Commands: - cancel Cancel one or more runs - get Show status, configuration, and timing details for a specific run - list List your active runs for the current profile (use --all-status for finished runs) - logs Stream or fetch logs for a run - register-image Mirror a Docker image into the workspace registry - run Submit a training workload from a YAML config + cancel Cancel one or more runs + convert-to-dabs Convert an AIR run YAML into a Databricks Asset Bundle + get Show status, configuration, and timing details for a specific run + list List your active runs for the current profile (use --all-status for finished runs) + logs Stream or fetch logs for a run + register-image Mirror a Docker image into the workspace registry + run Submit a training workload from a YAML config Flags: -h, --help help for air diff --git a/experimental/air/cmd/air.go b/experimental/air/cmd/air.go index fbf40a34b52..13b3ee18559 100644 --- a/experimental/air/cmd/air.go +++ b/experimental/air/cmd/air.go @@ -23,6 +23,7 @@ experimental and may change in future versions.`, cmd.AddCommand(newLogsCommand()) cmd.AddCommand(newCancelCommand()) cmd.AddCommand(newRegisterImageCommand()) + cmd.AddCommand(newConvertToDabsCommand()) return cmd } diff --git a/experimental/air/cmd/air_test.go b/experimental/air/cmd/air_test.go index 7efac253a2b..1843acfe900 100644 --- a/experimental/air/cmd/air_test.go +++ b/experimental/air/cmd/air_test.go @@ -14,7 +14,7 @@ func TestNewRegistersAllSubcommands(t *testing.T) { registered[c.Name()] = true } - want := []string{"run", "get", "list", "logs", "cancel", "register-image"} + want := []string{"run", "get", "list", "logs", "cancel", "register-image", "convert-to-dabs"} for _, name := range want { assert.True(t, registered[name], "subcommand %q is not registered", name) } diff --git a/experimental/air/cmd/convert_to_dabs.go b/experimental/air/cmd/convert_to_dabs.go new file mode 100644 index 00000000000..1ae604e4d19 --- /dev/null +++ b/experimental/air/cmd/convert_to_dabs.go @@ -0,0 +1,532 @@ +package aircmd + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + + "github.com/databricks/cli/cmd/root" + "github.com/databricks/cli/libs/cmdio" + "github.com/databricks/cli/libs/dyn" + "github.com/databricks/cli/libs/dyn/yamlsaver" + "github.com/spf13/cobra" + "go.yaml.in/yaml/v3" +) + +// convert_to_dabs turns an AIR CLI run YAML into a Databricks Asset Bundle so a +// workload authored for `air run` can be deployed and managed as a bundle. +// +// It is a purely local, syntactic translation: it maps the run config onto a +// schema-valid ai_runtime_task (the SDK jobs.AiRuntimeTask — experiment + +// deployments[].{command_path,compute} + code_source_path, with framework fields +// like retries/timeout on the surrounding task) and writes command.sh plus the +// env_vars.json / secret_env_vars.json / hyperparameters.yaml sidecars at the +// bundle root. It does NOT package, snapshot, or upload anything. +// +// code_source_path is emitted as the source directory relative to the bundle (the +// bundle root defaults to the YAML's directory, which contains it). At deploy the +// aicode mutator (bundle/config/mutator/aicode) packages that directory and uploads +// it — so convert never touches the code. Dependencies are folded into the job's +// environments[] spec, which the runtime installs from directly; no requirements.yaml +// is emitted. + +// dabsTargetName is the single default target emitted; a development-mode target +// is the conventional starting point for a generated bundle. +const dabsTargetName = "dev" + +func newConvertToDabsCommand() *cobra.Command { + var ( + outputDir string + force bool + ) + + cmd := &cobra.Command{ + Use: "convert-to-dabs ", + Args: root.ExactArgs(1), + Short: "Convert an AIR run YAML into a Databricks Asset Bundle", + Long: `Convert an AIR CLI run YAML config into a Databricks Asset Bundle (DABs). + +The emitted bundle can be deployed with the standard DABs workflow: + + databricks bundle validate + databricks bundle deploy + +bundle deploy uploads the code source and launch scripts for you, so no manual +upload step is required. This command performs a purely local translation and +does not contact the workspace.`, + } + + cmd.Flags().StringVar(&outputDir, "output-dir", "", "Directory to write the bundle into (default: a -bundle folder next to the input YAML). Accepts an absolute or relative path.") + cmd.Flags().BoolVar(&force, "force", false, "Overwrite the generated bundle files if they already exist.") + + cmd.RunE = func(cmd *cobra.Command, args []string) error { + ctx := cmd.Context() + yamlPath := args[0] + + cfg, err := loadRunConfig(yamlPath) + if err != nil { + return err + } + + // Default the bundle to the input YAML's directory. The bundle's sync root + // must contain the code_source so `code_source_path` resolves within it (the + // deploy-time aicode mutator packages the source in place), and root_path is + // resolved relative to the YAML, so the YAML's dir is the natural bundle root. + // An explicit --output-dir overrides. + dir := outputDir + if dir == "" { + dir = filepath.Dir(yamlPath) + } + + written, err := writeBundle(ctx, cfg, yamlPath, dir, force) + if err != nil { + return err + } + + printConvertNextSteps(ctx, dir, written, bundleResourceKey(cfg.ExperimentName), conversionNotes(cfg)) + return nil + } + + return cmd +} + +// convertToDabs builds the DABs bundle value and the loose launch artifacts for a +// run config. It reads only what the run path's buildArtifacts reads, so the +// mapping is unit-testable in isolation. Returns the bundle root as a +// map[string]dyn.Value (ready for yamlsaver) and the loose artifacts (command.sh + +// env/secret/param sidecars) to write at the bundle root. It does not touch the +// code_source; the deploy-time aicode mutator packages it in place. +func convertToDabs(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (map[string]dyn.Value, []uploadItem, error) { + // idempotency_token is intentionally not mapped: it dedups a single runs/submit + // call, which has no analogue for a persistent, repeatedly-runnable bundle job. + // + // usage_policy_name resolution is not ported (mirrors the submit path), and + // docker images have no ai_runtime_task representation yet. + if cfg.UsagePolicyName != nil { + return nil, nil, errors.New("usage_policy_name is not yet supported by convert-to-dabs") + } + if cfg.Environment != nil && cfg.Environment.DockerImage != nil { + return nil, nil, errors.New("environment.docker_image is not yet supported by convert-to-dabs") + } + if snap := codeSnapshot(cfg); snap != nil { + // remote_volume points the code archive at a specific UC Volume. The bundle's + // artifact location is set bundle-wide via workspace.artifact_path, not + // per-code-source, so a per-source Volume isn't representable here. + if snap.RemoteVolume != nil { + return nil, nil, errors.New("code_source.snapshot.remote_volume is not supported by convert-to-dabs; set workspace.artifact_path in the bundle instead") + } + // git pins to a committed revision, but convert packages nothing — the + // deploy-time mutator uploads the working tree as it is on disk. Deploying a + // specific revision therefore isn't supported; check it out before converting. + if snap.Git != nil { + return nil, nil, errors.New("code_source.snapshot.git is not supported by convert-to-dabs; deploy packages your working tree as-is, so check out the revision you want (git checkout ) before converting") + } + } + + codeSourcePath, err := bundleCodeSourcePath(ctx, cfg, configPath, bundleDir) + if err != nil { + return nil, nil, err + } + + artifacts, err := buildArtifacts(cfg, configPath) + if err != nil { + return nil, nil, err + } + // Drop requirements.yaml: the runtime installs pip deps from the job's + // environments[] spec (which convert populates), so a sidecar would be redundant. + artifacts = slices.DeleteFunc(artifacts, func(it uploadItem) bool { + return it.name == requirementsName + }) + + root := buildBundleValue(ctx, cfg, configPath, codeSourcePath) + return root, artifacts, nil +} + +// codeSnapshot returns the snapshot code source config, or nil if none. +func codeSnapshot(cfg *runConfig) *snapshotSourceConfig { + if cfg.CodeSource == nil { + return nil + } + return cfg.CodeSource.Snapshot +} + +// bundleCodeSourcePath resolves the code_source directory to a "./"-prefixed path +// relative to the bundle dir, for emission as ai_runtime_task.code_source_path. +// Returns "" when the config has no code_source. The path must be inside the bundle +// (the deploy-time mutator packages it in place and only handles in-bundle dirs). +func bundleCodeSourcePath(ctx context.Context, cfg *runConfig, configPath, bundleDir string) (string, error) { + snap := codeSnapshot(cfg) + if snap == nil { + return "", nil + } + root, err := resolveRootPath(ctx, snap.RootPath, filepath.Dir(configPath)) + if err != nil { + return "", err + } + bundleAbs, err := filepath.Abs(bundleDir) + if err != nil { + return "", err + } + rel, err := filepath.Rel(bundleAbs, root) + if err != nil || !filepath.IsLocal(rel) { + return "", fmt.Errorf("code_source root_path %q is not inside the bundle directory %q; run convert-to-dabs with --output-dir set to an ancestor of the code", snap.RootPath, bundleDir) + } + return localBundlePath(filepath.ToSlash(rel)), nil +} + +// nv builds a dyn.Value at "position" n: yamlsaver orders a map's keys by their +// Location line, so assigning ascending n values fixes the emitted key order. +// It routes through dyn.V so nested Go maps/slices are converted recursively, +// then stamps the ordering location. +func nv(v any, n int) dyn.Value { + return dyn.V(v).WithLocations([]dyn.Location{{Line: n}}) +} + +// localBundlePath renders a bundle-relative path with a leading "./" so bundle +// deploy classifies it as a local artifact to upload (see IsLibraryLocal). It is +// built from path.Join (forward slashes) so the emitted YAML is identical across +// operating systems. +func localBundlePath(p string) string { + return "./" + p +} + +// buildBundleValue assembles the bundle root as an ordered map[string]dyn.Value. +// codeSourcePath is the "./"-prefixed code_source dir relative to the bundle (empty +// when the config has no code_source); command.sh is a bundle-local artifact. +func buildBundleValue(ctx context.Context, cfg *runConfig, configPath, codeSourcePath string) map[string]dyn.Value { + name := cfg.ExperimentName + + // ai_runtime_task: experiment + one deployment (command_path + compute) + + // code_source_path. Only the fields the strict schema allows. + // + // command_path is "./"-prefixed so bundle deploy treats it as LOCAL and uploads + // it: libraries.IsLibraryLocal classifies a bare, extensionless path as a PyPI + // package name, which would deploy a path the backend can't resolve. + deployment := map[string]dyn.Value{ + "command_path": nv(localBundlePath(commandScriptName), 1), + "compute": nv(map[string]dyn.Value{ + "accelerator_type": nv(cfg.Compute.AcceleratorType, 1), + "accelerator_count": nv(cfg.Compute.NumAccelerators, 2), + }, 2), + } + + aiRuntimeTask := map[string]dyn.Value{ + "experiment": nv(name, 1), + "deployments": nv([]dyn.Value{dyn.V(deployment)}, 2), + } + line := 3 + if codeSourcePath != "" { + // The source dir relative to the bundle; the aicode mutator packages it at + // deploy and rewrites this field to the uploaded workspace path. + aiRuntimeTask["code_source_path"] = nv(codeSourcePath, line) + line++ + } + if cfg.MLflowRunName != nil { + aiRuntimeTask["mlflow_run"] = nv(*cfg.MLflowRunName, line) + line++ + } + if cfg.MLflowExperimentDirectory != nil { + aiRuntimeTask["mlflow_experiment_directory"] = nv(*cfg.MLflowExperimentDirectory, line) + } + + // Task wrapper: task_key + framework fields (retries/timeout) + env key + + // the ai_runtime_task. Framework fields live here per the schema, not inside + // ai_runtime_task. + task := map[string]dyn.Value{ + "task_key": nv(name, 1), + "environment_key": nv(aiRuntimeEnvironmentKey, 2), + } + taskLine := 3 + if cfg.MaxRetries != nil { + task["max_retries"] = nv(*cfg.MaxRetries, taskLine) + taskLine++ + } + if cfg.TimeoutMinutes != nil { + task["timeout_seconds"] = nv(cfg.timeoutSeconds(), taskLine) + taskLine++ + } + task["ai_runtime_task"] = nv(aiRuntimeTask, taskLine) + + // environments[]: version + the dependency set. The aicode.SynthesizeRequirements + // mutator regenerates requirements.yaml from this spec at deploy time, so the full + // dependency set (whether authored inline or in a requirements file) must live + // here — convert emits no requirements.yaml of its own. Resolve the version + // through the same path `air run` uses (config, else env override, else the + // default channel) so a config without an explicit version still pins the version + // the workload would have run with — not an empty spec. + envVersion, deps := bundleEnvironmentDeps(ctx, cfg, configPath) + envSpec := map[string]dyn.Value{ + "environment_version": nv(envVersion, 1), + } + if len(deps) > 0 { + depVals := make([]dyn.Value, len(deps)) + for i, d := range deps { + depVals[i] = dyn.V(d) + } + envSpec["dependencies"] = nv(depVals, 2) + } + environment := map[string]dyn.Value{ + "environment_key": nv(aiRuntimeEnvironmentKey, 1), + "spec": nv(envSpec, 2), + } + + job := map[string]dyn.Value{ + "name": nv(name, 1), + "tasks": nv([]dyn.Value{dyn.V(task)}, 2), + "environments": nv([]dyn.Value{dyn.V(environment)}, 3), + } + // usage_policy_id is an already-resolved budget policy id, so it maps directly + // to the job's budget_policy_id. (usage_policy_name needs server-side resolution + // and is rejected in convertToDabs.) + if cfg.UsagePolicyID != nil { + job["budget_policy_id"] = nv(*cfg.UsagePolicyID, 4) + } + if perms := buildPermissionsValue(cfg.Permissions); perms.Kind() != dyn.KindInvalid { + job["permissions"] = nv(perms.MustSequence(), 5) + } + + rootValue := map[string]dyn.Value{ + "bundle": nv(map[string]dyn.Value{ + "name": nv(name, 1), + }, 1), + "targets": nv(map[string]dyn.Value{ + dabsTargetName: nv(map[string]dyn.Value{ + "mode": nv("development", 1), + "default": nv(true, 2), + }, 1), + }, 2), + "resources": nv(map[string]dyn.Value{ + "jobs": nv(map[string]dyn.Value{ + bundleResourceKey(name): nv(job, 1), + }, 1), + }, 3), + } + return rootValue +} + +// bundleEnvironmentDeps resolves the runtime version and the flattened dependency +// list to emit in the bundle's environments[] spec. The aicode mutator synthesizes +// requirements.yaml from that spec at deploy, so the whole set must be here — +// whether the user authored dependencies inline or pointed at a requirements file. +// A requirements file is read and its non-comment, non-blank lines are inlined; the +// version, when the file carries one, wins over the config/default version. Any read +// error is best-effort ignored (writeBundle/buildArtifacts surface real problems); +// convert falls back to inline deps so the spec is never silently wrong. +func bundleEnvironmentDeps(ctx context.Context, cfg *runConfig, configPath string) (version string, deps []string) { + cfgVersion, _ := cfg.runtimeVersion() + version = dlRuntimeImage(ctx, cfgVersion) + + if inline, ok := cfg.inlineDependencies(); ok { + return version, inline + } + + reqPath, ok := cfg.requirementsFile() + if !ok { + return version, nil + } + if !filepath.IsAbs(reqPath) { + reqPath = filepath.Join(filepath.Dir(configPath), reqPath) + } + data, err := os.ReadFile(reqPath) + if err != nil { + return version, nil + } + // The requirements file is the same requirements.yaml shape the run path reads + // (version + dependencies), so parse it as such and inline the dependency lines. + var doc requirementsDoc + if err := yaml.Unmarshal(data, &doc); err != nil { + return version, nil + } + if doc.Version != "" { + version = dlRuntimeImage(ctx, doc.Version) + } + return version, doc.Dependencies +} + +// bundleResourceKey derives a job resource key from the experiment name. The key +// is emitted as an unquoted YAML map key, and DABs' strict loader rejects a key +// that parses as a non-string scalar (a purely numeric name like "12345" -> !!int, +// or "true"/"null"). experiment_name allows exactly [alphanumeric, -, _], so the +// only unsafe keys are those that YAML types as int/float/bool/null; prefix those +// with "job_" to force a string key. The human-facing name/experiment fields keep +// the original value (yamlsaver quotes them as scalar string values). +func bundleResourceKey(name string) string { + switch strings.ToLower(name) { + case "true", "false", "null": + return "job_" + name + } + if _, err := strconv.ParseFloat(name, 64); err == nil { + return "job_" + name + } + return name +} + +// buildPermissionsValue maps run-config permissions to DABs job permissions +// (level → principal). Returns an invalid value when there are none. +func buildPermissionsValue(perms []permission) dyn.Value { + if len(perms) == 0 { + return dyn.InvalidValue + } + out := make([]dyn.Value, 0, len(perms)) + for _, p := range perms { + m := map[string]dyn.Value{"level": nv(p.Level, 1)} + switch { + case p.UserName != nil: + m["user_name"] = nv(*p.UserName, 2) + case p.GroupName != nil: + m["group_name"] = nv(*p.GroupName, 2) + case p.ServicePrincipalName != nil: + m["service_principal_name"] = nv(*p.ServicePrincipalName, 2) + } + out = append(out, dyn.V(m)) + } + return dyn.V(out) +} + +// writeBundle writes the bundle into dir: databricks.yml plus the loose launch +// artifacts (command.sh + env/secret/param sidecars). It does not touch the code +// source — the deploy-time aicode mutator packages it in place. Unless force is set +// it refuses to overwrite existing files, so a re-run can't silently clobber a +// bundle the user has edited. Returns the relative paths written, for the +// next-steps message. +func writeBundle(ctx context.Context, cfg *runConfig, configPath, dir string, force bool) ([]string, error) { + root, artifacts, err := convertToDabs(ctx, cfg, configPath, dir) + if err != nil { + return nil, err + } + + // Restrict perms: the bundle carries env_vars.json (literal env var values), so + // keep the dir owner-only rather than world-readable. + if err := os.MkdirAll(dir, 0o700); err != nil { + return nil, err + } + + // Refuse to clobber an existing file, with one consistent message. A user + // re-running convert into the same dir gets a clear error rather than a silent + // overwrite of edits they may have made. The hint names --force rather than + // --output-dir: the code_source must live inside the bundle dir, so redirecting + // the output usually isn't a usable escape hatch for an in-place conversion. + writeFile := func(name string, data []byte) error { + if !force { + if _, err := os.Stat(filepath.Join(dir, name)); err == nil { + return fmt.Errorf("%s already exists in %s; pass --force to overwrite or remove it", name, dir) + } + } + return os.WriteFile(filepath.Join(dir, name), data, 0o600) + } + + bundlePath := filepath.Join(dir, "databricks.yml") + if !force { + if _, err := os.Stat(bundlePath); err == nil { + return nil, fmt.Errorf("databricks.yml already exists in %s; pass --force to overwrite or remove it", dir) + } + } + // SaveAsYAML's force arg is passed true unconditionally: the collision check + // above already decided whether overwriting is allowed. + if err := yamlsaver.NewSaver().SaveAsYAML(root, bundlePath, true); err != nil { + return nil, err + } + written := []string{"databricks.yml"} + + // Loose launch artifacts (command.sh + sidecars) at the bundle root. + for _, item := range artifacts { + if err := writeFile(item.name, item.data); err != nil { + return nil, fmt.Errorf("failed to write %s: %w", item.name, err) + } + written = append(written, item.name) + } + + return written, nil +} + +// printConvertNextSteps tells the user what was written and the exact deploy +// sequence, since the value of the command is a one-command deploy afterwards. +// +// It also spells out cleanup. This matters specifically for AIR users: `air run` +// submits an ephemeral runs/submit workload that the platform reaps on its own, +// whereas `bundle deploy` creates a *persistent* job that lingers until explicitly +// destroyed — DABs has no automatic GC. A user migrating from `air run` will not +// expect a durable resource, so we call out `bundle destroy` explicitly. +func printConvertNextSteps(ctx context.Context, dir string, written []string, jobKey string, notes []string) { + cmdio.LogString(ctx, fmt.Sprintf("Wrote a Databricks Asset Bundle to %s:", dir)) + for _, w := range written { + cmdio.LogString(ctx, " "+w) + } + + // Notes surface anything the user should know: fields we transformed or dropped, + // and values they may need to fill in. Migrating users otherwise can't tell what + // silently changed between their run YAML and the bundle. + if len(notes) > 0 { + cmdio.LogString(ctx, "") + cmdio.LogString(ctx, "Notes:") + for _, n := range notes { + cmdio.LogString(ctx, " - "+n) + } + } + + // Name this binary rather than a bare "databricks": ai_runtime_task is only + // understood by a CLI carrying it, and an older one on PATH drops the field with + // just a warning, deploying a job with no AI task at all. + self := cliInvocation() + + cmdio.LogString(ctx, "") + cmdio.LogString(ctx, "To deploy and run this workload as a bundle:") + cmdio.LogString(ctx, " 1. cd "+dir) + cmdio.LogString(ctx, " 2. "+self+" bundle validate") + cmdio.LogString(ctx, " 3. "+self+" bundle deploy") + cmdio.LogString(ctx, " 4. "+self+" bundle run "+jobKey) + cmdio.LogString(ctx, "") + cmdio.LogString(ctx, "bundle deploy uploads the code source and launch scripts automatically.") + cmdio.LogString(ctx, "") + cmdio.LogString(ctx, "Run these with this same CLI: a build without ai_runtime_task support") + cmdio.LogString(ctx, "only warns about the unknown field, then deploys a job with no AI task.") + cmdio.LogString(ctx, "") + cmdio.LogString(ctx, "Unlike `air run` (which submits an ephemeral run), bundle deploy creates a") + cmdio.LogString(ctx, "persistent job that is not garbage-collected. When you are done, remove the") + cmdio.LogString(ctx, "job and its uploaded files with:") + cmdio.LogString(ctx, " "+self+" bundle destroy") +} + +// cliInvocation is how the user should spell this binary in a follow-up command. +// A path-qualified argv[0] (./dbcli, ../dbcli) is kept as typed so copy-paste works +// from the same cwd; a bare name resolved via PATH is reported as "databricks". +func cliInvocation() string { + arg0 := os.Args[0] + if arg0 == "" { + return "databricks" + } + if arg0 == filepath.Base(arg0) { + return "databricks" + } + return arg0 +} + +// conversionNotes lists what the conversion staged out-of-band or could not +// represent natively, so a user migrating from `air run` sees what changed between +// their run YAML and the emitted bundle. +func conversionNotes(cfg *runConfig) []string { + var notes []string + + if codeSnapshot(cfg) != nil { + notes = append(notes, "code_source points at your source directory; bundle deploy packages and uploads it from there.") + } + + // env vars / secrets have no native ai_runtime_task field yet, so they ride as + // sidecar files the server-side launcher reads (same as `air run`). + if len(cfg.EnvVariables) > 0 { + notes = append(notes, "env_variables were written to env_vars.json (no native bundle field yet); they are uploaded with the code and applied at run time.") + } + if len(cfg.Secrets) > 0 { + notes = append(notes, "secrets were written to secret_env_vars.json (no native bundle field yet); they are resolved at run time.") + } + if len(cfg.Parameters) > 0 { + notes = append(notes, "parameters were written to hyperparameters.yaml; they are not a native bundle field and are passed through to the workload.") + } + + return notes +} diff --git a/experimental/air/cmd/convert_to_dabs_test.go b/experimental/air/cmd/convert_to_dabs_test.go new file mode 100644 index 00000000000..a9b8ed337a3 --- /dev/null +++ b/experimental/air/cmd/convert_to_dabs_test.go @@ -0,0 +1,433 @@ +package aircmd + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/databricks/cli/libs/dyn" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Re-running a conversion into a dir that already holds a generated bundle is +// refused by default (the user may have edited it) and allowed with --force. +func TestConvertToDabsForceOverwrite(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o700)) + cfg := "experiment_name: overwrite\ncommand: python train.py\n" + + "compute: {accelerator_type: GPU_1xH100, num_accelerators: 1}\n" + + "code_source:\n type: snapshot\n snapshot:\n root_path: ./src\n" + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(cfg), 0o600)) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + _, err = writeBundle(t.Context(), loaded, path, dir, false) + require.NoError(t, err) + + // A hand edit must survive a refused re-run. + edited := []byte("# hand-edited\n") + require.NoError(t, os.WriteFile(filepath.Join(dir, "databricks.yml"), edited, 0o600)) + + _, err = writeBundle(t.Context(), loaded, path, dir, false) + require.ErrorContains(t, err, "pass --force to overwrite") + kept, err := os.ReadFile(filepath.Join(dir, "databricks.yml")) + require.NoError(t, err) + assert.Equal(t, edited, kept) + + // With --force the generated bundle replaces it. + _, err = writeBundle(t.Context(), loaded, path, dir, true) + require.NoError(t, err) + regenerated, err := os.ReadFile(filepath.Join(dir, "databricks.yml")) + require.NoError(t, err) + assert.NotEqual(t, edited, regenerated) + assert.Contains(t, string(regenerated), "ai_runtime_task") +} + +func TestConvertToDabsCommandShape(t *testing.T) { + cmd := newConvertToDabsCommand() + assert.Equal(t, "convert-to-dabs ", cmd.Use) + assert.Empty(t, cmd.Commands(), "convert-to-dabs must not register subcommands") + // Exactly one positional (the YAML path). + assert.NoError(t, cmd.Args(cmd, []string{"run.yaml"})) + assert.Error(t, cmd.Args(cmd, []string{})) + assert.Error(t, cmd.Args(cmd, []string{"a", "b"})) +} + +// get is a small helper: read a dotted path out of the emitted bundle root. +func get(t *testing.T, root map[string]dyn.Value, path string) dyn.Value { + t.Helper() + v, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString(path)) + require.NoError(t, err, "path %q should exist", path) + return v +} + +func has(root map[string]dyn.Value, path string) bool { + _, err := dyn.GetByPath(dyn.V(root), dyn.MustPathFromString(path)) + return err == nil +} + +// A full config maps onto a schema-shaped bundle: bundle name, job/task keys, the +// ai_runtime_task (experiment + single deployment + code_source_path), framework +// fields on the task wrapper, and the environment spec. +func TestConvertToDabsFullMapping(t *testing.T) { + cfg := minimalConfig + ` +max_retries: 2 +timeout_minutes: 30 +mlflow_run_name: run-42 +code_source: + type: snapshot + snapshot: + root_path: ./src +environment: + version: 5 + dependencies: + - numpy + - torch +` + path := writeConfigFile(t, "run.yaml", cfg) + require.NoError(t, os.MkdirAll(filepath.Join(filepath.Dir(path), "src"), 0o700)) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, artifacts, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + + name := loaded.ExperimentName + assert.Equal(t, name, get(t, root, "bundle.name").MustString()) + assert.Equal(t, "development", get(t, root, "targets.dev.mode").MustString()) + + jobPath := "resources.jobs." + name + assert.Equal(t, name, get(t, root, jobPath+".name").MustString()) + + task := jobPath + ".tasks[0]" + assert.Equal(t, name, get(t, root, task+".task_key").MustString()) + // Framework fields live on the task wrapper, not in ai_runtime_task. + assert.Equal(t, int64(2), get(t, root, task+".max_retries").MustInt()) + assert.Equal(t, int64(1800), get(t, root, task+".timeout_seconds").MustInt()) + assert.False(t, has(root, task+".ai_runtime_task.max_retries"), "retries must not be inside ai_runtime_task") + + art := task + ".ai_runtime_task" + assert.Equal(t, name, get(t, root, art+".experiment").MustString()) + assert.Equal(t, "run-42", get(t, root, art+".mlflow_run").MustString()) + // code_source_path is the source dir relative to the bundle; the deploy-time + // aicode mutator packages it in place. + assert.Equal(t, "./src", get(t, root, art+".code_source_path").MustString()) + + dep := art + ".deployments[0]" + assert.Equal(t, "./"+commandScriptName, get(t, root, dep+".command_path").MustString()) + assert.Equal(t, "GPU_1xH100", get(t, root, dep+".compute.accelerator_type").MustString()) + assert.Equal(t, int64(1), get(t, root, dep+".compute.accelerator_count").MustInt()) + + env := jobPath + ".environments[0]" + assert.Equal(t, "default", get(t, root, env+".environment_key").MustString()) + assert.Equal(t, "5", get(t, root, env+".spec.environment_version").MustString()) + deps := get(t, root, env+".spec.dependencies").MustSequence() + require.Len(t, deps, 2) + assert.Equal(t, "numpy", deps[0].MustString()) + + // command.sh is always an artifact. requirements.yaml is NOT emitted: the + // deploy-time aicode.SynthesizeRequirements mutator regenerates it from the + // environments[] spec (asserted above), so convert must not also write it. + assert.Contains(t, itemNames(artifacts), commandScriptName) + assert.NotContains(t, itemNames(artifacts), requirementsName) +} + +// Optional fields are omitted rather than emitted empty: no code_source means no +// code_source_path; unset retries/timeout means no wrapper fields. +func TestConvertToDabsOmitsUnsetFields(t *testing.T) { + path := writeConfigFile(t, "run.yaml", minimalConfig) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, _, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + + name := loaded.ExperimentName + task := "resources.jobs." + name + ".tasks[0]" + assert.False(t, has(root, task+".max_retries")) + assert.False(t, has(root, task+".timeout_seconds")) + assert.False(t, has(root, task+".ai_runtime_task.code_source_path")) + assert.False(t, has(root, task+".ai_runtime_task.mlflow_run")) + // The command still needs a home even without code_source. + assert.Equal(t, "./"+commandScriptName, get(t, root, task+".ai_runtime_task.deployments[0].command_path").MustString()) + + // Even with no environment block, the default runtime version is pinned (what + // `air run` would have used) rather than emitting an empty environment spec. + env := "resources.jobs." + name + ".environments[0]" + assert.Equal(t, "4", get(t, root, env+".spec.environment_version").MustString()) + assert.False(t, has(root, env+".spec.dependencies")) +} + +// A DATABRICKS_DL_RUNTIME_IMAGE env override flows through the same resolution +// `air run` uses, so a converted bundle pins the same version. +func TestConvertToDabsRuntimeVersionEnvOverride(t *testing.T) { + t.Setenv(dlRuntimeImageEnv, "CLIENT-GPU-7") + path := writeConfigFile(t, "run.yaml", minimalConfig) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, _, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + + env := "resources.jobs." + loaded.ExperimentName + ".environments[0]" + assert.Equal(t, "7", get(t, root, env+".spec.environment_version").MustString()) +} + +// remote_volume can't be honored by a converted bundle (bundle deploy owns the +// artifact upload location), so it is rejected rather than silently ignored. +func TestConvertToDabsRejectsRemoteVolume(t *testing.T) { + cfg := minimalConfig + ` +code_source: + type: snapshot + snapshot: + root_path: ./src + remote_volume: /Volumes/main/default/code +` + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + _, _, err = convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.ErrorContains(t, err, "remote_volume is not supported") +} + +// env_variables / secrets / parameters ride as sidecar files (the ai_runtime_task +// proto has no inline fields for them), matching the CLI's own launch layout. +func TestConvertToDabsStagesEnvAndSecretSidecars(t *testing.T) { + cfg := minimalConfig + ` +env_variables: + FOO: bar +secrets: + TOKEN: scope/key +parameters: + lr: 0.1 +` + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, artifacts, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + + names := itemNames(artifacts) + assert.Contains(t, names, envVarsName) + assert.Contains(t, names, secretEnvVarsName) + assert.Contains(t, names, hyperparametersName) + + // They are NOT smuggled into ai_runtime_task (which would fail bundle validate). + name := loaded.ExperimentName + art := "resources.jobs." + name + ".tasks[0].ai_runtime_task" + assert.False(t, has(root, art+".env_variables")) + assert.False(t, has(root, art+".secrets")) +} + +// code_source_path is emitted as the source dir relative to the bundle and no code +// is copied — the deploy-time mutator packages it in place. writeBundle produces +// only databricks.yml + launch artifacts, not a code_source copy. +func TestConvertToDabsDoesNotCopyCode(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o700)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "src", "train.py"), []byte("print()\n"), 0o600)) + + cfg := "experiment_name: wt\ncommand: cd \"$CODE_SOURCE_PATH\" && python train.py\n" + + "compute: {accelerator_type: GPU_1xH100, num_accelerators: 1}\n" + + "code_source:\n type: snapshot\n snapshot:\n root_path: ./src\n" + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(cfg), 0o600)) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + written, err := writeBundle(t.Context(), loaded, path, dir, false) + require.NoError(t, err) + + // code_source_path points at the existing source dir; nothing is copied. + root, _, err := convertToDabs(t.Context(), loaded, path, dir) + require.NoError(t, err) + art := "resources.jobs." + loaded.ExperimentName + ".tasks[0].ai_runtime_task" + assert.Equal(t, "./src", get(t, root, art+".code_source_path").MustString()) + assert.NotContains(t, written, "code_source/") +} + +// A code_source root_path outside the bundle directory is rejected: the mutator only +// packages a directory inside the bundle sync root. +func TestConvertToDabsRejectsCodeOutsideBundle(t *testing.T) { + outside := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(outside, "src"), 0o700)) + + cfg := "experiment_name: outside\ncommand: python train.py\n" + + "compute: {accelerator_type: GPU_1xH100, num_accelerators: 1}\n" + + "code_source:\n type: snapshot\n snapshot:\n root_path: " + filepath.Join(outside, "src") + "\n" + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + // Bundle dir is the config's temp dir; the source is in a different tree. + _, _, err = convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.ErrorContains(t, err, "not inside the bundle") +} + +// A git-pinned code_source is rejected: convert no longer materializes a commit; +// the deploy-time mutator packages the working tree in place. +func TestConvertToDabsRejectsGitPin(t *testing.T) { + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "src"), 0o700)) + cfg := "experiment_name: git-pin\ncommand: python train.py\n" + + "compute: {accelerator_type: GPU_1xH100, num_accelerators: 1}\n" + + "code_source:\n type: snapshot\n snapshot:\n root_path: ./src\n git:\n commit: abc123\n" + path := filepath.Join(dir, "run.yaml") + require.NoError(t, os.WriteFile(path, []byte(cfg), 0o600)) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + _, _, err = convertToDabs(t.Context(), loaded, path, dir) + require.ErrorContains(t, err, "git is not supported") +} + +// A requirements-FILE dependency set (environment.dependencies is a path) is folded +// into the environments[] spec so the deploy-time aicode mutator can regenerate +// requirements.yaml from it. Convert emits no requirements.yaml artifact of its own. +func TestConvertToDabsFoldsRequirementsFileIntoEnvSpec(t *testing.T) { + dir := t.TempDir() + reqPath := filepath.Join(dir, "requirements.yaml") + require.NoError(t, os.WriteFile(reqPath, []byte("version: \"6\"\ndependencies:\n - numpy\n - pandas\n"), 0o600)) + + cfg := "experiment_name: reqfile\ncommand: python train.py\n" + + "compute: {accelerator_type: GPU_1xH100, num_accelerators: 1}\n" + + "environment:\n dependencies: " + reqPath + "\n" + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, artifacts, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + + env := "resources.jobs." + loaded.ExperimentName + ".environments[0]" + assert.Equal(t, "6", get(t, root, env+".spec.environment_version").MustString()) + deps := get(t, root, env+".spec.dependencies").MustSequence() + require.Len(t, deps, 2) + assert.Equal(t, "numpy", deps[0].MustString()) + assert.Equal(t, "pandas", deps[1].MustString()) + + // No requirements.yaml artifact: the mutator regenerates it from the spec. + assert.NotContains(t, itemNames(artifacts), requirementsName) +} + +// conversionNotes surfaces what was transformed/staged so a migrating user knows +// what changed between their run YAML and the bundle. +func TestConvertToDabsConversionNotes(t *testing.T) { + cfg := minimalConfig + ` +env_variables: {FOO: bar} +secrets: {TOKEN: scope/key} +parameters: {lr: 0.1} +code_source: + type: snapshot + snapshot: + root_path: ./src +` + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + notes := conversionNotes(loaded) + joined := strings.Join(notes, "\n") + assert.Contains(t, joined, "code_source") // source-dir behavior + assert.Contains(t, joined, "env_vars.json") // env vars staged + assert.Contains(t, joined, "secret_env_vars.json") // secrets staged + assert.Contains(t, joined, "hyperparameters.yaml") // parameters staged + + // A minimal config with none of those has no notes. + base := writeConfigFile(t, "min.yaml", minimalConfig) + minCfg, err := loadRunConfig(base) + require.NoError(t, err) + assert.Empty(t, conversionNotes(minCfg)) +} + +// usage_policy_id is a resolved budget policy id and maps to the job's +// budget_policy_id (usage_policy_name, which needs resolution, is rejected). +func TestConvertToDabsMapsUsagePolicyID(t *testing.T) { + cfg := minimalConfig + "usage_policy_id: budget-abc-123\n" + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, _, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + assert.Equal(t, "budget-abc-123", get(t, root, "resources.jobs."+loaded.ExperimentName+".budget_policy_id").MustString()) +} + +func TestConvertToDabsMapsPermissions(t *testing.T) { + cfg := minimalConfig + ` +permissions: + - user_name: alice@example.com + level: CAN_MANAGE + - group_name: eng + level: CAN_VIEW +` + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + + root, _, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + + perms := get(t, root, "resources.jobs."+loaded.ExperimentName+".permissions").MustSequence() + require.Len(t, perms, 2) + assert.Equal(t, "CAN_MANAGE", perms[0].Get("level").MustString()) + assert.Equal(t, "alice@example.com", perms[0].Get("user_name").MustString()) + assert.Equal(t, "eng", perms[1].Get("group_name").MustString()) +} + +// A numeric or reserved-word experiment_name would be an unquoted YAML map key +// that DABs' strict loader rejects (!!int / !!bool). The job resource key is +// prefixed to stay a string, while name/experiment keep the original value. +func TestConvertToDabsSafeJobKey(t *testing.T) { + cases := map[string]string{ + "12345": "job_12345", + "1.5e3": "job_1.5e3", + "true": "job_true", + "null": "job_null", + } + for name, wantKey := range cases { + assert.Equal(t, wantKey, bundleResourceKey(name), "key for %q", name) + } + // A normal name is used as-is. + assert.Equal(t, "my-run_1", bundleResourceKey("my-run_1")) + + // End to end: a numeric name lands under the prefixed key, but name/experiment + // keep the numeric string value. + cfg := "experiment_name: \"12345\"\ncommand: python t.py\n" + + "compute: {accelerator_type: GPU_1xH100, num_accelerators: 1}\n" + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + root, _, err := convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.NoError(t, err) + assert.Equal(t, "12345", get(t, root, "resources.jobs.job_12345.name").MustString()) + assert.Equal(t, "12345", get(t, root, "resources.jobs.job_12345.tasks[0].ai_runtime_task.experiment").MustString()) +} + +func TestConvertToDabsRejectsUnsupported(t *testing.T) { + t.Run("docker_image", func(t *testing.T) { + cfg := minimalConfig + ` +environment: + docker_image: + url: myregistry/img:tag +` + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + _, _, err = convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.ErrorContains(t, err, "docker_image is not yet supported") + }) + + t.Run("usage_policy_name", func(t *testing.T) { + cfg := minimalConfig + "usage_policy_name: my-policy\n" + path := writeConfigFile(t, "run.yaml", cfg) + loaded, err := loadRunConfig(path) + require.NoError(t, err) + _, _, err = convertToDabs(t.Context(), loaded, path, filepath.Dir(path)) + require.ErrorContains(t, err, "usage_policy_name is not yet supported") + }) +} diff --git a/experimental/air/cmd/snapshot_package.go b/experimental/air/cmd/snapshot_package.go index 672366086c9..b81f33a65b1 100644 --- a/experimental/air/cmd/snapshot_package.go +++ b/experimental/air/cmd/snapshot_package.go @@ -42,9 +42,24 @@ func createGitArchiveSnapshot(ctx context.Context, git gitRepo, commitSHA, outpu // excluded; a .gitignore at repoPath is honored. func createPlainTarball(ctx context.Context, repoPath, outputTarball string, includePaths []string) error { dirName := filepath.Base(repoPath) - parent := filepath.Dir(repoPath) + // Absolute so it resolves correctly regardless of tar's working dir (set below). + parent, err := filepath.Abs(filepath.Dir(repoPath)) + if err != nil { + return err + } + + // Pass the archive path relative to its own directory (run tar there), never a + // full path: on Windows an absolute path like `C:\out\x.tar.gz` makes tar read + // the `C:` as a remote host ("Cannot connect to C:"), since tar treats a colon + // in the -f arg as host:path. A bare basename with -C avoids that on GNU tar and + // bsdtar alike. + outDirAbs, err := filepath.Abs(filepath.Dir(outputTarball)) + if err != nil { + return err + } + outName := filepath.Base(outputTarball) - args := []string{"-czf", outputTarball} + args := []string{"-czf", outName} // Exclude macOS AppleDouble files: they sort before the real top-level dir and // hijack a remote `head -1` parse. No-op on Linux. @@ -68,7 +83,9 @@ func createPlainTarball(ctx context.Context, repoPath, outputTarball string, inc } // Archive from the parent so the directory name is preserved; with include_paths, - // prefix each so entries nest under it (matching git archive --prefix). + // prefix each so entries nest under it (matching git archive --prefix). -C only + // affects the file operands that follow it, not the -f archive path (which + // resolves against tar's working dir, set to outDirAbs below). args = append(args, "-C", parent) if len(includePaths) > 0 { for _, p := range includePaths { @@ -79,6 +96,8 @@ func createPlainTarball(ctx context.Context, repoPath, outputTarball string, inc } cmd := exec.CommandContext(ctx, "tar", args...) + // Run tar in the output directory so the bare -f basename lands there. + cmd.Dir = outDirAbs var stderr bytes.Buffer cmd.Stderr = &stderr if err := cmd.Run(); err != nil {