diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index 8eac3866e1..142035af42 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -7,6 +7,7 @@ To disable this, set the environment variable DATABRICKS_CACHE_ENABLED to false. ### CLI +* Add commands to pipelines command group ([#4275](https://github.com/databricks/cli/pull/4275)) ### Bundles * Enable caching user identity by default ([#4202](https://github.com/databricks/cli/pull/4202)) diff --git a/acceptance/acceptance_test.go b/acceptance/acceptance_test.go index a41efc4fa0..12fb35b4a8 100644 --- a/acceptance/acceptance_test.go +++ b/acceptance/acceptance_test.go @@ -216,7 +216,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int { repls.SetPath(execPath, "[CLI]") pipelinesPath := filepath.Join(buildDir, "pipelines") + exeSuffix - err = copyFile(execPath, pipelinesPath) + if _, err := os.Stat(pipelinesPath); err == nil { + err := os.Remove(pipelinesPath) + require.NoError(t, err) + } + err = os.Symlink(execPath, pipelinesPath) require.NoError(t, err) t.Setenv("PIPELINES", pipelinesPath) repls.SetPath(pipelinesPath, "[PIPELINES]") diff --git a/acceptance/pipelines/databricks-cli-help-disabled/output.txt b/acceptance/pipelines/databricks-cli-help-disabled/output.txt deleted file mode 100644 index fb8a4eca90..0000000000 --- a/acceptance/pipelines/databricks-cli-help-disabled/output.txt +++ /dev/null @@ -1,49 +0,0 @@ -The Lakeflow Spark Declarative Pipelines API allows you to create, edit, - delete, start, and view details about pipelines. - - Spark Declarative Pipelines is a framework for building reliable, - maintainable, and testable data processing pipelines. You define the - transformations to perform on your data, and Spark Declarative Pipelines - manages task orchestration, cluster management, monitoring, data quality, and - error handling. - - Instead of defining your data pipelines using a series of separate Apache - Spark tasks, Spark Declarative Pipelines manages how your data is transformed - based on a target schema you define for each processing step. You can also - enforce data quality with Spark Declarative Pipelines expectations. - Expectations allow you to define expected data quality and specify how to - handle records that fail those expectations. - -Usage: - databricks pipelines [flags] - databricks pipelines [command] - -Available Commands - clone Clone a pipeline. - create Create a pipeline. - delete Delete a pipeline. - get Get a pipeline. - get-update Get a pipeline update. - list-pipeline-events List pipeline events. - list-pipelines List pipelines. - list-updates List pipeline updates. - start-update Start a pipeline. - stop Stop a pipeline. - update Edit a pipeline. - -Permission Commands - get-permission-levels Get pipeline permission levels. - get-permissions Get pipeline permissions. - set-permissions Set pipeline permissions. - update-permissions Update pipeline permissions. - -Flags: - -h, --help help for pipelines - -Global Flags: - --debug enable debug logging - -o, --output type output type: text or json (default text) - -p, --profile string ~/.databrickscfg profile - -t, --target string bundle target to use (if applicable) - -Use "databricks pipelines [command] --help" for more information about a command. diff --git a/acceptance/pipelines/databricks-cli-help-disabled/script b/acceptance/pipelines/databricks-cli-help-disabled/script deleted file mode 100644 index 9facb3eee2..0000000000 --- a/acceptance/pipelines/databricks-cli-help-disabled/script +++ /dev/null @@ -1 +0,0 @@ -$CLI pipelines diff --git a/acceptance/pipelines/databricks-cli-help/script b/acceptance/pipelines/databricks-cli-help/script index e7c7b15a36..9facb3eee2 100644 --- a/acceptance/pipelines/databricks-cli-help/script +++ b/acceptance/pipelines/databricks-cli-help/script @@ -1 +1 @@ -ENABLE_PIPELINES_CLI=1 $CLI pipelines +$CLI pipelines diff --git a/acceptance/pipelines/deploy/auto-approve/out.test.toml b/acceptance/pipelines/deploy/auto-approve/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/deploy/auto-approve/out.test.toml +++ b/acceptance/pipelines/deploy/auto-approve/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/deploy/auto-approve/output.txt b/acceptance/pipelines/deploy/auto-approve/output.txt index a7ef416cf5..b33d6396da 100644 --- a/acceptance/pipelines/deploy/auto-approve/output.txt +++ b/acceptance/pipelines/deploy/auto-approve/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... Deploying resources... Updating deployment state... @@ -10,7 +10,7 @@ View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] >>> rm resources.yml === Try to redeploy without --auto-approve - should fail ->>> errcode [PIPELINES] deploy +>>> errcode [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the @@ -24,7 +24,7 @@ Error: the deployment requires destructive actions, but current console does not Exit code: 1 === Redeploy with --auto-approve - should succeed ->>> [PIPELINES] deploy --auto-approve +>>> [ENTRYPOINT] deploy --auto-approve Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files... This action will result in the deletion or recreation of the following Lakeflow Spark Declarative Pipelines along with the diff --git a/acceptance/pipelines/deploy/auto-approve/script b/acceptance/pipelines/deploy/auto-approve/script index 579031b617..65587a4b97 100644 --- a/acceptance/pipelines/deploy/auto-approve/script +++ b/acceptance/pipelines/deploy/auto-approve/script @@ -1,10 +1,10 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy title "Remove resources from configuration to test auto-approve" trace rm resources.yml title "Try to redeploy without --auto-approve - should fail" -trace errcode $PIPELINES deploy +trace errcode $ENTRYPOINT deploy title "Redeploy with --auto-approve - should succeed" -trace $PIPELINES deploy --auto-approve +trace $ENTRYPOINT deploy --auto-approve diff --git a/acceptance/pipelines/deploy/create-pipeline/out.test.toml b/acceptance/pipelines/deploy/create-pipeline/out.test.toml index d560f1de04..301b1ab5d9 100644 --- a/acceptance/pipelines/deploy/create-pipeline/out.test.toml +++ b/acceptance/pipelines/deploy/create-pipeline/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$PIPELINES"] diff --git a/acceptance/pipelines/deploy/create-pipeline/script b/acceptance/pipelines/deploy/create-pipeline/script index 3042dea425..62ebd74961 100644 --- a/acceptance/pipelines/deploy/create-pipeline/script +++ b/acceptance/pipelines/deploy/create-pipeline/script @@ -1,3 +1,3 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id') trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}" diff --git a/acceptance/pipelines/deploy/create-pipeline/test.toml b/acceptance/pipelines/deploy/create-pipeline/test.toml new file mode 100644 index 0000000000..ba0ebcbc1a --- /dev/null +++ b/acceptance/pipelines/deploy/create-pipeline/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$PIPELINES"] diff --git a/acceptance/pipelines/deploy/fail-on-active-runs/out.test.toml b/acceptance/pipelines/deploy/fail-on-active-runs/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/deploy/fail-on-active-runs/out.test.toml +++ b/acceptance/pipelines/deploy/fail-on-active-runs/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/deploy/fail-on-active-runs/output.txt b/acceptance/pipelines/deploy/fail-on-active-runs/output.txt index afd3bd913a..113e2262d6 100644 --- a/acceptance/pipelines/deploy/fail-on-active-runs/output.txt +++ b/acceptance/pipelines/deploy/fail-on-active-runs/output.txt @@ -1,12 +1,12 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-fail-on-active-runs/default/files... Deploying resources... Updating deployment state... Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] ->>> errcode [PIPELINES] deploy --fail-on-active-runs +>>> [ENTRYPOINT] deploy --fail-on-active-runs Error: pipeline [UUID] is running diff --git a/acceptance/pipelines/deploy/fail-on-active-runs/script b/acceptance/pipelines/deploy/fail-on-active-runs/script index 3894cb7d7a..ac202c5595 100644 --- a/acceptance/pipelines/deploy/fail-on-active-runs/script +++ b/acceptance/pipelines/deploy/fail-on-active-runs/script @@ -1,4 +1,4 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy # We deploy the bundle again to check that the deploy is failing if the job is running -trace errcode $PIPELINES deploy --fail-on-active-runs +errcode trace $ENTRYPOINT deploy --fail-on-active-runs diff --git a/acceptance/pipelines/deploy/force-lock/out.test.toml b/acceptance/pipelines/deploy/force-lock/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/deploy/force-lock/out.test.toml +++ b/acceptance/pipelines/deploy/force-lock/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/deploy/force-lock/output.txt b/acceptance/pipelines/deploy/force-lock/output.txt index 890301f964..7b98988ce8 100644 --- a/acceptance/pipelines/deploy/force-lock/output.txt +++ b/acceptance/pipelines/deploy/force-lock/output.txt @@ -3,7 +3,7 @@ >>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock === test deployment without force-lock (should fail) ->>> errcode [PIPELINES] deploy +>>> errcode [ENTRYPOINT] deploy Error: Failed to acquire deployment lock: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +0000 UTC. Use --force-lock to override Error: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +0000 UTC. Use --force-lock to override @@ -11,7 +11,7 @@ Error: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +000 Exit code: 1 === test deployment with force-lock (should succeed) ->>> [PIPELINES] deploy --force-lock +>>> [ENTRYPOINT] deploy --force-lock Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/files... Deploying resources... Updating deployment state... diff --git a/acceptance/pipelines/deploy/force-lock/script b/acceptance/pipelines/deploy/force-lock/script index 83e4a100af..d41846ab7b 100644 --- a/acceptance/pipelines/deploy/force-lock/script +++ b/acceptance/pipelines/deploy/force-lock/script @@ -4,7 +4,7 @@ title "upload lock file" trace $CLI workspace import /Workspace/Users/$username/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock title "test deployment without force-lock (should fail)" -trace errcode $PIPELINES deploy +trace errcode $ENTRYPOINT deploy title "test deployment with force-lock (should succeed)" -trace $PIPELINES deploy --force-lock +trace $ENTRYPOINT deploy --force-lock diff --git a/acceptance/pipelines/deploy/oss-spark-error/out.test.toml b/acceptance/pipelines/deploy/oss-spark-error/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/deploy/oss-spark-error/out.test.toml +++ b/acceptance/pipelines/deploy/oss-spark-error/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/deploy/oss-spark-error/output.txt b/acceptance/pipelines/deploy/oss-spark-error/output.txt index 1e31fbcea9..609de417f1 100644 --- a/acceptance/pipelines/deploy/oss-spark-error/output.txt +++ b/acceptance/pipelines/deploy/oss-spark-error/output.txt @@ -1,6 +1,6 @@ === Fail to deploy OSS Spark pipelines-formatted YAML file ->>> musterr [PIPELINES] deploy +>>> musterr [ENTRYPOINT] deploy Error: [TEST_TMP_DIR]/oss-pipeline.yml seems to be formatted for open-source Spark Declarative Pipelines. Pipelines CLI currently only supports Lakeflow Spark Declarative Pipelines development. To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init". diff --git a/acceptance/pipelines/deploy/oss-spark-error/script b/acceptance/pipelines/deploy/oss-spark-error/script index 4b9071c54b..c2810eaeec 100644 --- a/acceptance/pipelines/deploy/oss-spark-error/script +++ b/acceptance/pipelines/deploy/oss-spark-error/script @@ -1,2 +1,2 @@ title "Fail to deploy OSS Spark pipelines-formatted YAML file" -trace musterr $PIPELINES deploy +trace musterr $ENTRYPOINT deploy diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml b/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt b/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt index 08adc39d9e..0b46bad8c0 100644 --- a/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Warning: unknown field: unknown_property at resources.pipelines.test-pipeline in databricks.yml:8:7 diff --git a/acceptance/pipelines/deploy/render-diagnostics-warning/script b/acceptance/pipelines/deploy/render-diagnostics-warning/script index 735145a6c9..2883564f8f 100644 --- a/acceptance/pipelines/deploy/render-diagnostics-warning/script +++ b/acceptance/pipelines/deploy/render-diagnostics-warning/script @@ -1,3 +1,3 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy # print newline to comply with whitespace linter printf "\n" diff --git a/acceptance/pipelines/deploy/test.toml b/acceptance/pipelines/deploy/test.toml new file mode 100644 index 0000000000..d5fac16851 --- /dev/null +++ b/acceptance/pipelines/deploy/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/deploy/var-flag/out.test.toml b/acceptance/pipelines/deploy/var-flag/out.test.toml index d560f1de04..301b1ab5d9 100644 --- a/acceptance/pipelines/deploy/var-flag/out.test.toml +++ b/acceptance/pipelines/deploy/var-flag/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$PIPELINES"] diff --git a/acceptance/pipelines/deploy/var-flag/script b/acceptance/pipelines/deploy/var-flag/script index 13e016eaed..358bf47366 100644 --- a/acceptance/pipelines/deploy/var-flag/script +++ b/acceptance/pipelines/deploy/var-flag/script @@ -1,5 +1,5 @@ title "Test --var flag usage" -trace $PIPELINES deploy --var="catalog=custom_catalog" --var="schema=custom_schema" --auto-approve +trace $ENTRYPOINT deploy --var="catalog=custom_catalog" --var="schema=custom_schema" --auto-approve title "Verify: Check that variables were substituted correctly" PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id') diff --git a/acceptance/pipelines/deploy/var-flag/test.toml b/acceptance/pipelines/deploy/var-flag/test.toml new file mode 100644 index 0000000000..ba0ebcbc1a --- /dev/null +++ b/acceptance/pipelines/deploy/var-flag/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$PIPELINES"] diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/out.test.toml b/acceptance/pipelines/dry-run/dry-run-pipeline/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/dry-run/dry-run-pipeline/out.test.toml +++ b/acceptance/pipelines/dry-run/dry-run-pipeline/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt index 55e3a849f8..e61aaee2a4 100644 --- a/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt +++ b/acceptance/pipelines/dry-run/dry-run-pipeline/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === Dry running pipeline, should have validate_only set to true ->>> [PIPELINES] dry-run +>>> [ENTRYPOINT] dry-run Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] @@ -20,7 +20,7 @@ Update ID: [UUID] } === Dry running pipeline with KEY, expect same output as without KEY ->>> [PIPELINES] dry-run my_pipeline +>>> [ENTRYPOINT] dry-run my_pipeline Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] diff --git a/acceptance/pipelines/dry-run/dry-run-pipeline/script b/acceptance/pipelines/dry-run/dry-run-pipeline/script index 6c1af6c7e6..4969ecb7d1 100755 --- a/acceptance/pipelines/dry-run/dry-run-pipeline/script +++ b/acceptance/pipelines/dry-run/dry-run-pipeline/script @@ -3,12 +3,12 @@ print_requests() { rm out.requests.txt } -trace $PIPELINES deploy +trace $ENTRYPOINT deploy rm out.requests.txt title "Dry running pipeline, should have validate_only set to true" -trace $PIPELINES dry-run +trace $ENTRYPOINT dry-run print_requests title "Dry running pipeline with KEY, expect same output as without KEY" -trace $PIPELINES dry-run my_pipeline +trace $ENTRYPOINT dry-run my_pipeline print_requests diff --git a/acceptance/pipelines/dry-run/no-wait/out.test.toml b/acceptance/pipelines/dry-run/no-wait/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/dry-run/no-wait/out.test.toml +++ b/acceptance/pipelines/dry-run/no-wait/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/dry-run/no-wait/output.txt b/acceptance/pipelines/dry-run/no-wait/output.txt index ab087a346d..21fa36b676 100644 --- a/acceptance/pipelines/dry-run/no-wait/output.txt +++ b/acceptance/pipelines/dry-run/no-wait/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === Dry running pipeline with --no-wait flag ->>> [PIPELINES] dry-run --no-wait +>>> [ENTRYPOINT] dry-run --no-wait Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] diff --git a/acceptance/pipelines/dry-run/no-wait/script b/acceptance/pipelines/dry-run/no-wait/script index f2d4ade6a9..8ab101002e 100644 --- a/acceptance/pipelines/dry-run/no-wait/script +++ b/acceptance/pipelines/dry-run/no-wait/script @@ -1,3 +1,3 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy title "Dry running pipeline with --no-wait flag" -trace $PIPELINES dry-run --no-wait +trace $ENTRYPOINT dry-run --no-wait diff --git a/acceptance/pipelines/dry-run/restart/out.test.toml b/acceptance/pipelines/dry-run/restart/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/dry-run/restart/out.test.toml +++ b/acceptance/pipelines/dry-run/restart/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/dry-run/restart/output.txt b/acceptance/pipelines/dry-run/restart/output.txt index 4520e92d69..426378cb2c 100644 --- a/acceptance/pipelines/dry-run/restart/output.txt +++ b/acceptance/pipelines/dry-run/restart/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === Dry running pipeline with --restart flag, should stop the current pipeline and start a new run ->>> [PIPELINES] dry-run --restart +>>> [ENTRYPOINT] dry-run --restart Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] diff --git a/acceptance/pipelines/dry-run/restart/script b/acceptance/pipelines/dry-run/restart/script index d15135a72c..8e02183ae6 100644 --- a/acceptance/pipelines/dry-run/restart/script +++ b/acceptance/pipelines/dry-run/restart/script @@ -3,9 +3,9 @@ print_requests() { rm out.requests.txt } -trace $PIPELINES deploy +trace $ENTRYPOINT deploy rm out.requests.txt title "Dry running pipeline with --restart flag, should stop the current pipeline and start a new run" -trace $PIPELINES dry-run --restart +trace $ENTRYPOINT dry-run --restart trace print_requests diff --git a/acceptance/pipelines/dry-run/test.toml b/acceptance/pipelines/dry-run/test.toml new file mode 100644 index 0000000000..d5fac16851 --- /dev/null +++ b/acceptance/pipelines/dry-run/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/init/error-cases/out.test.toml b/acceptance/pipelines/init/error-cases/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/init/error-cases/out.test.toml +++ b/acceptance/pipelines/init/error-cases/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/init/error-cases/output.txt b/acceptance/pipelines/init/error-cases/output.txt index 3209536177..6aa6d85a5f 100644 --- a/acceptance/pipelines/init/error-cases/output.txt +++ b/acceptance/pipelines/init/error-cases/output.txt @@ -1,12 +1,12 @@ === Test with invalid project name (contains uppercase letters) ->>> errcode [PIPELINES] init --config-file ./invalid_input.json --output-dir invalid-output +>>> errcode [ENTRYPOINT] init --config-file ./invalid_input.json --output-dir invalid-output Error: failed to load config from file ./invalid_input.json: invalid value for project_name: "InvalidProjectName". Name must consist of lower case letters, numbers, and underscores. Exit code: 1 === Test with non-existent config file ->>> errcode [PIPELINES] init --config-file ./nonexistent.json --output-dir invalid-output-2 +>>> errcode [ENTRYPOINT] init --config-file ./nonexistent.json --output-dir invalid-output-2 Error: failed to load config from file ./nonexistent.json: open ./nonexistent.json: no such file or directory Exit code: 1 diff --git a/acceptance/pipelines/init/error-cases/script b/acceptance/pipelines/init/error-cases/script index c1bb876058..2806f88d9d 100644 --- a/acceptance/pipelines/init/error-cases/script +++ b/acceptance/pipelines/init/error-cases/script @@ -1,5 +1,5 @@ title "Test with invalid project name (contains uppercase letters)" -trace errcode $PIPELINES init --config-file ./invalid_input.json --output-dir invalid-output +trace errcode $ENTRYPOINT init --config-file ./invalid_input.json --output-dir invalid-output title "Test with non-existent config file" -trace errcode $PIPELINES init --config-file ./nonexistent.json --output-dir invalid-output-2 +trace errcode $ENTRYPOINT init --config-file ./nonexistent.json --output-dir invalid-output-2 diff --git a/acceptance/pipelines/init/python/out.test.toml b/acceptance/pipelines/init/python/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/init/python/out.test.toml +++ b/acceptance/pipelines/init/python/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/init/python/output.txt b/acceptance/pipelines/init/python/output.txt index 283980248e..09887e2183 100644 --- a/acceptance/pipelines/init/python/output.txt +++ b/acceptance/pipelines/init/python/output.txt @@ -1,6 +1,6 @@ === Test basic pipelines init with configuration file ->>> [PIPELINES] init --config-file ./input.json --output-dir output +>>> [ENTRYPOINT] init --config-file ./input.json --output-dir output Welcome to the template for Lakeflow Spark Declarative Pipelines! Please answer the below to tailor your project to your preferences. diff --git a/acceptance/pipelines/init/python/script b/acceptance/pipelines/init/python/script index d87149f126..937401f69b 100644 --- a/acceptance/pipelines/init/python/script +++ b/acceptance/pipelines/init/python/script @@ -1,4 +1,4 @@ title "Test basic pipelines init with configuration file" -trace $PIPELINES init --config-file ./input.json --output-dir output +trace $ENTRYPOINT init --config-file ./input.json --output-dir output mv output/my_python_project/.gitignore output/my_python_project/out.gitignore diff --git a/acceptance/pipelines/init/sql/out.test.toml b/acceptance/pipelines/init/sql/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/init/sql/out.test.toml +++ b/acceptance/pipelines/init/sql/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/init/sql/output.txt b/acceptance/pipelines/init/sql/output.txt index 972048e65a..43cd7bee5e 100644 --- a/acceptance/pipelines/init/sql/output.txt +++ b/acceptance/pipelines/init/sql/output.txt @@ -1,6 +1,6 @@ === Test pipelines init with SQL configuration ->>> [PIPELINES] init --config-file ./input.json --output-dir output +>>> [ENTRYPOINT] init --config-file ./input.json --output-dir output Welcome to the template for Lakeflow Spark Declarative Pipelines! Please answer the below to tailor your project to your preferences. diff --git a/acceptance/pipelines/init/sql/script b/acceptance/pipelines/init/sql/script index e1e7deb3fe..48a2181165 100644 --- a/acceptance/pipelines/init/sql/script +++ b/acceptance/pipelines/init/sql/script @@ -1,5 +1,5 @@ title "Test pipelines init with SQL configuration" -trace $PIPELINES init --config-file ./input.json --output-dir output +trace $ENTRYPOINT init --config-file ./input.json --output-dir output # Do not affect this repository's git behaviour mv output/my_sql_project/.gitignore output/my_sql_project/out.gitignore diff --git a/acceptance/pipelines/init/test.toml b/acceptance/pipelines/init/test.toml new file mode 100644 index 0000000000..d5fac16851 --- /dev/null +++ b/acceptance/pipelines/init/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/open/databricks.yml b/acceptance/pipelines/open/completion/databricks.yml similarity index 100% rename from acceptance/pipelines/open/databricks.yml rename to acceptance/pipelines/open/completion/databricks.yml diff --git a/acceptance/pipelines/databricks-cli-help-disabled/out.test.toml b/acceptance/pipelines/open/completion/out.test.toml similarity index 100% rename from acceptance/pipelines/databricks-cli-help-disabled/out.test.toml rename to acceptance/pipelines/open/completion/out.test.toml diff --git a/acceptance/pipelines/open/completion/output.txt b/acceptance/pipelines/open/completion/output.txt new file mode 100644 index 0000000000..d90862e9de --- /dev/null +++ b/acceptance/pipelines/open/completion/output.txt @@ -0,0 +1,5 @@ + +>>> [PIPELINES] __complete open , +test-pipelines-open +:4 +Completion ended with directive: ShellCompDirectiveNoFileComp diff --git a/acceptance/pipelines/open/completion/script b/acceptance/pipelines/open/completion/script new file mode 100644 index 0000000000..6058f495a5 --- /dev/null +++ b/acceptance/pipelines/open/completion/script @@ -0,0 +1 @@ +trace $PIPELINES __complete open , diff --git a/acceptance/pipelines/open/open-after-deployment/databricks.yml b/acceptance/pipelines/open/open-after-deployment/databricks.yml new file mode 100644 index 0000000000..737ce5f4bc --- /dev/null +++ b/acceptance/pipelines/open/open-after-deployment/databricks.yml @@ -0,0 +1,7 @@ +bundle: + name: pipelines-open + +resources: + pipelines: + test-pipelines-open: + name: test-pipelines-open diff --git a/acceptance/pipelines/open/open b/acceptance/pipelines/open/open-after-deployment/open similarity index 100% rename from acceptance/pipelines/open/open rename to acceptance/pipelines/open/open-after-deployment/open diff --git a/acceptance/pipelines/open/out.test.toml b/acceptance/pipelines/open/open-after-deployment/out.test.toml similarity index 73% rename from acceptance/pipelines/open/out.test.toml rename to acceptance/pipelines/open/open-after-deployment/out.test.toml index 216969a761..c2d3d8086a 100644 --- a/acceptance/pipelines/open/out.test.toml +++ b/acceptance/pipelines/open/open-after-deployment/out.test.toml @@ -7,3 +7,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/open/output.txt b/acceptance/pipelines/open/open-after-deployment/output.txt similarity index 77% rename from acceptance/pipelines/open/output.txt rename to acceptance/pipelines/open/open-after-deployment/output.txt index 86a0a06e1a..33555d4264 100644 --- a/acceptance/pipelines/open/output.txt +++ b/acceptance/pipelines/open/open-after-deployment/output.txt @@ -1,11 +1,11 @@ === not deployed yet ->>> [PIPELINES] open +>>> [ENTRYPOINT] open Error: pipeline does not have a URL associated with it (has it been deployed?) Exit code: 1 ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipelines-open/default/files... Deploying resources... Updating deployment state... @@ -14,17 +14,11 @@ View your pipeline test-pipelines-open here: [DATABRICKS_URL]/pipelines/[UUID]?o === Modify PATH so that real open is not run === open after deployment. This will fail to open browser and complain, that's ok, we only want the message ->>> [PIPELINES] open +>>> [ENTRYPOINT] open Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] Error: exec: "open": cannot run executable found relative to current directory === open with KEY, expect same output as opening without KEY ->>> [PIPELINES] open test-pipelines-open +>>> [ENTRYPOINT] open test-pipelines-open Opening browser at [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] Error: exec: "open": cannot run executable found relative to current directory - -=== test auto-completion handler ->>> [PIPELINES] __complete open , -test-pipelines-open -:4 -Completion ended with directive: ShellCompDirectiveNoFileComp diff --git a/acceptance/pipelines/open/script b/acceptance/pipelines/open/open-after-deployment/script similarity index 56% rename from acceptance/pipelines/open/script rename to acceptance/pipelines/open/open-after-deployment/script index 1e1871902d..f53a12d4cc 100644 --- a/acceptance/pipelines/open/script +++ b/acceptance/pipelines/open/open-after-deployment/script @@ -1,16 +1,13 @@ title "not deployed yet" -errcode trace $PIPELINES open +errcode trace $ENTRYPOINT open -errcode trace $PIPELINES deploy +errcode trace $ENTRYPOINT deploy title "Modify PATH so that real open is not run" export PATH=.:$PATH title "open after deployment. This will fail to open browser and complain, that's ok, we only want the message" -musterr trace $PIPELINES open +musterr trace $ENTRYPOINT open title "open with KEY, expect same output as opening without KEY" -musterr trace $PIPELINES open test-pipelines-open - -title "test auto-completion handler" -trace $PIPELINES __complete open , +musterr trace $ENTRYPOINT open test-pipelines-open diff --git a/acceptance/pipelines/open/open-after-deployment/test.toml b/acceptance/pipelines/open/open-after-deployment/test.toml new file mode 100644 index 0000000000..edd282ef76 --- /dev/null +++ b/acceptance/pipelines/open/open-after-deployment/test.toml @@ -0,0 +1,5 @@ +GOOS.windows = false +GOOS.linux = false + +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/open/test.toml b/acceptance/pipelines/open/test.toml deleted file mode 100644 index 078e52c97e..0000000000 --- a/acceptance/pipelines/open/test.toml +++ /dev/null @@ -1,2 +0,0 @@ -GOOS.windows = false -GOOS.linux = false diff --git a/acceptance/pipelines/run/completion/databricks.yml b/acceptance/pipelines/run/completion/databricks.yml new file mode 100644 index 0000000000..04d0fead10 --- /dev/null +++ b/acceptance/pipelines/run/completion/databricks.yml @@ -0,0 +1,11 @@ +bundle: + name: test-pipeline-run + +resources: + pipelines: + my_pipeline: + name: test-pipeline + # job is runnable, but doesn't impact auto-selection of pipeline + jobs: + my_job: + name: "My Job" diff --git a/acceptance/pipelines/run/completion/out.test.toml b/acceptance/pipelines/run/completion/out.test.toml new file mode 100644 index 0000000000..301b1ab5d9 --- /dev/null +++ b/acceptance/pipelines/run/completion/out.test.toml @@ -0,0 +1,6 @@ +Local = true +Cloud = false + +[EnvMatrix] + DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$PIPELINES"] diff --git a/acceptance/pipelines/run/completion/output.txt b/acceptance/pipelines/run/completion/output.txt new file mode 100644 index 0000000000..843b5b530d --- /dev/null +++ b/acceptance/pipelines/run/completion/output.txt @@ -0,0 +1,5 @@ + +>>> [PIPELINES] __complete run , +my_pipeline +:4 +Completion ended with directive: ShellCompDirectiveNoFileComp diff --git a/acceptance/pipelines/run/completion/script b/acceptance/pipelines/run/completion/script new file mode 100644 index 0000000000..afd1547c41 --- /dev/null +++ b/acceptance/pipelines/run/completion/script @@ -0,0 +1 @@ +trace $PIPELINES __complete run , diff --git a/acceptance/pipelines/run/completion/test.toml b/acceptance/pipelines/run/completion/test.toml new file mode 100644 index 0000000000..ba0ebcbc1a --- /dev/null +++ b/acceptance/pipelines/run/completion/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$PIPELINES"] diff --git a/acceptance/pipelines/run/no-wait/out.test.toml b/acceptance/pipelines/run/no-wait/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/run/no-wait/out.test.toml +++ b/acceptance/pipelines/run/no-wait/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/run/no-wait/output.txt b/acceptance/pipelines/run/no-wait/output.txt index 790f0aa63e..b9b822dd4c 100644 --- a/acceptance/pipelines/run/no-wait/output.txt +++ b/acceptance/pipelines/run/no-wait/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === test --no-wait flag ->>> [PIPELINES] run --no-wait +>>> [ENTRYPOINT] run --no-wait Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] diff --git a/acceptance/pipelines/run/no-wait/script b/acceptance/pipelines/run/no-wait/script index 9325073d4c..8701b201ec 100644 --- a/acceptance/pipelines/run/no-wait/script +++ b/acceptance/pipelines/run/no-wait/script @@ -1,3 +1,3 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy title "test --no-wait flag" -trace $PIPELINES run --no-wait +trace $ENTRYPOINT run --no-wait diff --git a/acceptance/pipelines/run/refresh-flags/out.test.toml b/acceptance/pipelines/run/refresh-flags/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/run/refresh-flags/out.test.toml +++ b/acceptance/pipelines/run/refresh-flags/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/run/refresh-flags/output.txt b/acceptance/pipelines/run/refresh-flags/output.txt index 36e906594f..69f696ed14 100644 --- a/acceptance/pipelines/run/refresh-flags/output.txt +++ b/acceptance/pipelines/run/refresh-flags/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === Running pipeline with --refresh flag and specific tables ->>> [PIPELINES] run --refresh table1,table2 +>>> [ENTRYPOINT] run --refresh table1,table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] @@ -30,7 +30,7 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh-all flag ->>> [PIPELINES] run --full-refresh-all +>>> [ENTRYPOINT] run --full-refresh-all Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] @@ -50,7 +50,7 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh flag and specific tables ->>> [PIPELINES] run --full-refresh table1,table2 +>>> [ENTRYPOINT] run --full-refresh table1,table2 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] @@ -73,7 +73,7 @@ Pipeline configurations for this update: } === Running pipeline with --full-refresh flag and --refresh flag ->>> [PIPELINES] run --full-refresh table1,table2 --refresh table3,table4 +>>> [ENTRYPOINT] run --full-refresh table1,table2 --refresh table3,table4 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] diff --git a/acceptance/pipelines/run/refresh-flags/script b/acceptance/pipelines/run/refresh-flags/script index d24262e89d..9d4b5c70dc 100644 --- a/acceptance/pipelines/run/refresh-flags/script +++ b/acceptance/pipelines/run/refresh-flags/script @@ -3,21 +3,21 @@ print_requests() { rm out.requests.txt } -trace $PIPELINES deploy +trace $ENTRYPOINT deploy rm out.requests.txt title "Running pipeline with --refresh flag and specific tables" -trace $PIPELINES run --refresh table1,table2 +trace $ENTRYPOINT run --refresh table1,table2 trace print_requests title "Running pipeline with --full-refresh-all flag" -trace $PIPELINES run --full-refresh-all +trace $ENTRYPOINT run --full-refresh-all trace print_requests title "Running pipeline with --full-refresh flag and specific tables" -trace $PIPELINES run --full-refresh table1,table2 +trace $ENTRYPOINT run --full-refresh table1,table2 trace print_requests title "Running pipeline with --full-refresh flag and --refresh flag" -trace $PIPELINES run --full-refresh table1,table2 --refresh table3,table4 +trace $ENTRYPOINT run --full-refresh table1,table2 --refresh table3,table4 trace print_requests diff --git a/acceptance/pipelines/run/restart/out.test.toml b/acceptance/pipelines/run/restart/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/run/restart/out.test.toml +++ b/acceptance/pipelines/run/restart/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/run/restart/output.txt b/acceptance/pipelines/run/restart/output.txt index c99683e4d8..d34d9cc37d 100644 --- a/acceptance/pipelines/run/restart/output.txt +++ b/acceptance/pipelines/run/restart/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === Running pipeline with --restart flag ->>> [PIPELINES] run --restart +>>> [ENTRYPOINT] run --restart Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] diff --git a/acceptance/pipelines/run/restart/script b/acceptance/pipelines/run/restart/script index b67281dc97..67b6276543 100644 --- a/acceptance/pipelines/run/restart/script +++ b/acceptance/pipelines/run/restart/script @@ -3,10 +3,10 @@ print_requests() { rm out.requests.txt } -trace $PIPELINES deploy +trace $ENTRYPOINT deploy rm out.requests.txt title "Running pipeline with --restart flag" # Should stop the current pipeline and start a new update -trace $PIPELINES run --restart +trace $ENTRYPOINT run --restart trace print_requests diff --git a/acceptance/pipelines/run/run-info/out.test.toml b/acceptance/pipelines/run/run-info/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/run/run-info/out.test.toml +++ b/acceptance/pipelines/run/run-info/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/run/run-info/output.txt b/acceptance/pipelines/run/run-info/output.txt index 55c892f2fd..6566c2d980 100644 --- a/acceptance/pipelines/run/run-info/output.txt +++ b/acceptance/pipelines/run/run-info/output.txt @@ -1,5 +1,5 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/default/files... Deploying resources... Updating deployment state... @@ -7,7 +7,7 @@ Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] === Run pipeline and display pipeline configuration summary and events durations ->>> [PIPELINES] run +>>> [ENTRYPOINT] run Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/test-update-123 [TIMESTAMP] update_progress "Update test-update-123 is COMPLETED." diff --git a/acceptance/pipelines/run/run-info/script b/acceptance/pipelines/run/run-info/script index 464c55caed..fbed7c0db7 100644 --- a/acceptance/pipelines/run/run-info/script +++ b/acceptance/pipelines/run/run-info/script @@ -1,7 +1,7 @@ -trace $PIPELINES deploy +trace $ENTRYPOINT deploy title "Run pipeline and display pipeline configuration summary and events durations" # Test note: The mock API handler for "GET /api/2.0/pipelines/{pipeline_id}/events" doesn't support # the OrderBy query parameter. In production, the real API returns events in descending order (newest first), and # bundle/run/pipeline.go reverses them again to display chronologically. # So, test output appears "backwards" compared to production behavior. -trace $PIPELINES run +trace $ENTRYPOINT run diff --git a/acceptance/pipelines/run/run-pipeline/out.test.toml b/acceptance/pipelines/run/run-pipeline/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/run/run-pipeline/out.test.toml +++ b/acceptance/pipelines/run/run-pipeline/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/run/run-pipeline/output.txt b/acceptance/pipelines/run/run-pipeline/output.txt index 711799b4ec..ceabf45d91 100644 --- a/acceptance/pipelines/run/run-pipeline/output.txt +++ b/acceptance/pipelines/run/run-pipeline/output.txt @@ -1,36 +1,7 @@ ->>> [PIPELINES] deploy -Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/default/files... -Deploying resources... -Updating deployment state... -Deployment complete! -View your job my_job here: [DATABRICKS_URL]/jobs/[NUMID]?o=[NUMID] -View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] - === Run pipeline ->>> [PIPELINES] run -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] - -Update ID: [UUID] - -Update for pipeline completed successfully. - -Pipeline configurations for this update: -• All tables are refreshed - -=== Run pipeline with KEY, expect same output as without KEY ->>> [PIPELINES] run my_pipeline -Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] - -Update ID: [UUID] - -Update for pipeline completed successfully. +>>> [ENTRYPOINT] run +Error: resource not found or not yet deployed. Did you forget to run 'databricks bundle deploy'? -Pipeline configurations for this update: -• All tables are refreshed -=== completion handler ->>> [PIPELINES] __complete run , -my_pipeline -:4 -Completion ended with directive: ShellCompDirectiveNoFileComp +Exit code: 1 diff --git a/acceptance/pipelines/run/run-pipeline/script b/acceptance/pipelines/run/run-pipeline/script index b816724bad..b26ab67e80 100755 --- a/acceptance/pipelines/run/run-pipeline/script +++ b/acceptance/pipelines/run/run-pipeline/script @@ -1,9 +1,5 @@ -trace $PIPELINES deploy title "Run pipeline" -trace $PIPELINES run +trace $ENTRYPOINT run title "Run pipeline with KEY, expect same output as without KEY" -trace $PIPELINES run my_pipeline - -title "completion handler" -trace $PIPELINES __complete run , +trace $ENTRYPOINT run my_pipeline diff --git a/acceptance/pipelines/run/run-pipeline/test.toml b/acceptance/pipelines/run/run-pipeline/test.toml new file mode 100644 index 0000000000..d5fac16851 --- /dev/null +++ b/acceptance/pipelines/run/run-pipeline/test.toml @@ -0,0 +1,2 @@ +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/run/test.toml b/acceptance/pipelines/run/test.toml new file mode 100644 index 0000000000..8fd32d44ac --- /dev/null +++ b/acceptance/pipelines/run/test.toml @@ -0,0 +1,4 @@ +Ignore = ['.databricks'] + +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/stop/out.test.toml b/acceptance/pipelines/stop/out.test.toml index d560f1de04..876b628373 100644 --- a/acceptance/pipelines/stop/out.test.toml +++ b/acceptance/pipelines/stop/out.test.toml @@ -3,3 +3,4 @@ Cloud = false [EnvMatrix] DATABRICKS_BUNDLE_ENGINE = ["terraform", "direct"] + ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/acceptance/pipelines/stop/output.txt b/acceptance/pipelines/stop/output.txt index e15f087fc3..cf8a1843bd 100644 --- a/acceptance/pipelines/stop/output.txt +++ b/acceptance/pipelines/stop/output.txt @@ -1,12 +1,12 @@ ->>> [PIPELINES] deploy +>>> [ENTRYPOINT] deploy Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-stop/default/files... Deploying resources... Updating deployment state... Deployment complete! View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID] ->>> [PIPELINES] run +>>> [ENTRYPOINT] run Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID] Update ID: [UUID] @@ -17,7 +17,7 @@ Pipeline configurations for this update: • All tables are refreshed === Stopping pipeline ->>> [PIPELINES] stop +>>> [ENTRYPOINT] stop Stopping my_pipeline... my_pipeline has been stopped. { @@ -26,7 +26,7 @@ my_pipeline has been stopped. } === Stopping pipeline with KEY, expect same output as without KEY ->>> [PIPELINES] stop my_pipeline +>>> [ENTRYPOINT] stop my_pipeline Stopping my_pipeline... my_pipeline has been stopped. { diff --git a/acceptance/pipelines/stop/script b/acceptance/pipelines/stop/script index efdd9243d9..d9881cb752 100644 --- a/acceptance/pipelines/stop/script +++ b/acceptance/pipelines/stop/script @@ -3,14 +3,14 @@ print_requests() { rm out.requests.txt } -trace $PIPELINES deploy -trace $PIPELINES run +trace $ENTRYPOINT deploy +trace $ENTRYPOINT run rm out.requests.txt title "Stopping pipeline" -trace $PIPELINES stop +trace $ENTRYPOINT stop print_requests title "Stopping pipeline with KEY, expect same output as without KEY" -trace $PIPELINES stop my_pipeline +trace $ENTRYPOINT stop my_pipeline print_requests diff --git a/acceptance/pipelines/stop/test.toml b/acceptance/pipelines/stop/test.toml index 159efe0269..29ba77fc30 100644 --- a/acceptance/pipelines/stop/test.toml +++ b/acceptance/pipelines/stop/test.toml @@ -1 +1,4 @@ RecordRequests = true + +[EnvMatrix] +ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"] diff --git a/cmd/pipelines/deploy.go b/cmd/pipelines/deploy.go index 1f95fd48c5..d966a962d3 100644 --- a/cmd/pipelines/deploy.go +++ b/cmd/pipelines/deploy.go @@ -44,12 +44,13 @@ func deployCommand() *cobra.Command { b.Config.Bundle.Deployment.FailOnActiveRuns = failOnActiveRuns } }, - Verbose: verbose, - AlwaysPull: true, - FastValidate: true, - Build: true, - Deploy: true, - IsPipelinesCLI: true, + Verbose: verbose, + SkipInitContext: false, + AlwaysPull: true, + FastValidate: true, + Build: true, + Deploy: true, + IsPipelinesCLI: true, }) if err != nil { return err diff --git a/cmd/pipelines/pipelines.go b/cmd/pipelines/pipelines.go index 6a56cdd732..e5f4e971d0 100644 --- a/cmd/pipelines/pipelines.go +++ b/cmd/pipelines/pipelines.go @@ -2,8 +2,6 @@ package pipelines import ( "context" - "os" - "slices" "github.com/databricks/cli/cmd/pipelines/root" "github.com/spf13/cobra" @@ -19,7 +17,7 @@ func New(ctx context.Context) *cobra.Command { cli.AddCommand(dryRunCommand()) cli.AddCommand(authCommand()) cli.AddCommand(destroyCommand()) - cli.AddCommand(StopCommand()) + cli.AddCommand(StopCommand(StopCommandOpts{SkipInitContext: false})) cli.AddCommand(historyCommand()) cli.AddCommand(logsCommand()) cli.AddCommand(versionCommand()) @@ -30,16 +28,6 @@ func New(ctx context.Context) *cobra.Command { // that are separate from main CLI commands defined in Commands. const ManagementGroupID = "management" -// Enabled disables Pipelines CLI in Databricks CLI until it's ready -func Enabled() bool { - value, ok := os.LookupEnv("ENABLE_PIPELINES_CLI") - if !ok { - return false - } - - return slices.Contains([]string{"1", "true", "t", "yes"}, value) -} - // Commands returns the list of commands that are shared between // the standalone pipelines CLI and databricks pipelines. // Note: auth and version are excluded as they are only for standalone CLI. diff --git a/cmd/pipelines/stop.go b/cmd/pipelines/stop.go index 143a300992..426f7b1a3e 100644 --- a/cmd/pipelines/stop.go +++ b/cmd/pipelines/stop.go @@ -36,7 +36,13 @@ func resolveStopArgument(ctx context.Context, b *bundle.Bundle, args []string) ( return "", errors.New("expected a KEY of the pipeline to stop") } -func StopCommand() *cobra.Command { +type StopCommandOpts struct { + // SkipInitContext skips logdiag.InitContext, and can be used if it's already initialized + // since stop command is reused + SkipInitContext bool +} + +func StopCommand(opts StopCommandOpts) *cobra.Command { cmd := &cobra.Command{ Use: "stop [KEY]", Short: "Stop a pipeline", @@ -50,6 +56,7 @@ If there is only one pipeline in the project, KEY is optional and the pipeline w var key string b, err := utils.ProcessBundle(cmd, utils.ProcessOptions{ ErrorOnEmptyState: true, + SkipInitContext: opts.SkipInitContext, PostInitFunc: func(ctx context.Context, b *bundle.Bundle) error { var err error key, err = resolveStopArgument(ctx, b, args) diff --git a/cmd/workspace/pipelines/overrides.go b/cmd/workspace/pipelines/overrides.go index f8a7ed9ad8..08c36deabe 100644 --- a/cmd/workspace/pipelines/overrides.go +++ b/cmd/workspace/pipelines/overrides.go @@ -10,10 +10,6 @@ import ( ) func init() { - if !pipelinesCli.Enabled() { - return - } - cmdOverrides = append(cmdOverrides, func(cli *cobra.Command) { // all auto-generated commands apart from nonManagementCommands go into 'management' group nonManagementCommands := []string{ @@ -55,7 +51,12 @@ func init() { return originalRunE(cmd, args) } // Looks like a bundle key or no args - use Lakeflow stop - return pipelinesCli.StopCommand().RunE(cmd, args) + + // context is already initialized by workspace command group + // if we initialize it again, CLI crashes + opts := pipelinesCli.StopCommandOpts{SkipInitContext: true} + + return pipelinesCli.StopCommand(opts).RunE(cmd, args) } // Update usage to reflect dual nature