diff --git a/acceptance/bundle/config-remote-sync/config_edits/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/config_edits/databricks.yml.tmpl index d7a7aa4d75..9c04adb336 100644 --- a/acceptance/bundle/config-remote-sync/config_edits/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/config_edits/databricks.yml.tmpl @@ -34,6 +34,7 @@ targets: env: dev version: v1 team: data-team + description: "original description" max_concurrent_runs: 1 environments: - environment_key: default diff --git a/acceptance/bundle/config-remote-sync/config_edits/output.txt b/acceptance/bundle/config-remote-sync/config_edits/output.txt index ff88716b3c..b0a2d91b18 100644 --- a/acceptance/bundle/config-remote-sync/config_edits/output.txt +++ b/acceptance/bundle/config-remote-sync/config_edits/output.txt @@ -14,7 +14,9 @@ Deployment complete! === Case 5: Updated locally, removed remotely -=== Case 6: Added locally and remotely with same value (no drift expected) +=== Case 6: Removed locally, edited remotely + +=== Case 7: Added locally and remotely with same value (no drift expected) === Edit job remotely @@ -22,10 +24,12 @@ Deployment complete! Detected changes in 1 resource(s): Resource: resources.jobs.my_job + description: add email_notifications.on_failure: replace email_notifications.on_failure[0]: replace max_concurrent_runs: replace tags['env']: remove + timeout_seconds: remove @@ -34,22 +38,30 @@ Resource: resources.jobs.my_job >>> diff.py databricks.yml.backup databricks.yml --- databricks.yml.backup +++ databricks.yml -@@ -24,5 +24,5 @@ +@@ -13,4 +13,5 @@ + node_type_id: [NODE_TYPE_ID] + num_workers: 1 ++ description: updated remotely + + targets: +@@ -23,5 +24,5 @@ - success@example.com on_failure: - - config-failure@example.com + - remote-failure@example.com parameters: - name: catalog -@@ -35,7 +35,6 @@ +@@ -34,9 +35,7 @@ unit: DAYS tags: - env: config-production team: data-team + project: analytics - max_concurrent_runs: 3 +- timeout_seconds: 3600 + max_concurrent_runs: 5 - timeout_seconds: 3600 environments: + - environment_key: default >>> [CLI] bundle destroy --auto-approve The following resources will be deleted: diff --git a/acceptance/bundle/config-remote-sync/config_edits/script b/acceptance/bundle/config-remote-sync/config_edits/script index e7defc9b65..6612cc0b72 100755 --- a/acceptance/bundle/config-remote-sync/config_edits/script +++ b/acceptance/bundle/config-remote-sync/config_edits/script @@ -82,22 +82,38 @@ read -r -d '' case5 <<'EOF' || true r["tags"].pop("env", None) EOF -title "Case 6: Added locally and remotely with same value (no drift expected)" +title "Case 6: Removed locally, edited remotely" echo old=$(cat <<'EOF' - my_job: - email_notifications: + description: "original description" + max_concurrent_runs: 3 EOF ) new=$(cat <<'EOF' - my_job: - description: A test job - email_notifications: + max_concurrent_runs: 3 EOF ) update_file.py databricks.yml "$old" "$new" read -r -d '' case6 <<'EOF' || true -r["description"] = "A test job" +r["description"] = "updated remotely" +EOF + +title "Case 7: Added locally and remotely with same value (no drift expected)" +echo +old=$(cat <<'EOF' + team: data-team +EOF +) +new=$(cat <<'EOF' + team: data-team + project: analytics +EOF +) +update_file.py databricks.yml "$old" "$new" +read -r -d '' case7 <<'EOF' || true +if "tags" not in r: + r["tags"] = {} +r["tags"]["project"] = "analytics" EOF title "Edit job remotely" @@ -114,6 +130,8 @@ $case4 $case5 $case6 + +$case7 EOF title "Detect and save changes" diff --git a/acceptance/bundle/config-remote-sync/job_fields/databricks.yml.tmpl b/acceptance/bundle/config-remote-sync/job_fields/databricks.yml.tmpl index 00adce9a2c..2fd1ea7a7d 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/databricks.yml.tmpl +++ b/acceptance/bundle/config-remote-sync/job_fields/databricks.yml.tmpl @@ -4,6 +4,7 @@ bundle: resources: jobs: my_job: + performance_target: STANDARD email_notifications: on_success: - success@example.com diff --git a/acceptance/bundle/config-remote-sync/job_fields/output.txt b/acceptance/bundle/config-remote-sync/job_fields/output.txt index bb5fd3279e..3a66dc23e2 100644 --- a/acceptance/bundle/config-remote-sync/job_fields/output.txt +++ b/acceptance/bundle/config-remote-sync/job_fields/output.txt @@ -13,6 +13,7 @@ Resource: resources.jobs.my_job email_notifications.no_alert_for_skipped_runs: add email_notifications.on_failure: add parameters: replace + performance_target: remove tags['team']: add trigger.pause_status: add trigger.periodic: remove @@ -25,7 +26,11 @@ Resource: resources.jobs.my_job >>> diff.py databricks.yml.backup databricks.yml --- databricks.yml.backup +++ databricks.yml -@@ -8,13 +8,19 @@ +@@ -5,17 +5,22 @@ + jobs: + my_job: +- performance_target: STANDARD + email_notifications: on_success: - success@example.com + no_alert_for_skipped_runs: true @@ -52,7 +57,7 @@ Resource: resources.jobs.my_job + - samples.nyctaxi.trips environments: - environment_key: default -@@ -31,4 +37,6 @@ +@@ -32,4 +37,6 @@ node_type_id: [NODE_TYPE_ID] num_workers: 1 + tags: diff --git a/acceptance/bundle/config-remote-sync/job_fields/script b/acceptance/bundle/config-remote-sync/job_fields/script index 6822fa350f..381fb0db69 100755 --- a/acceptance/bundle/config-remote-sync/job_fields/script +++ b/acceptance/bundle/config-remote-sync/job_fields/script @@ -19,6 +19,7 @@ r["email_notifications"]["on_failure"] = ["failure@example.com"] r["email_notifications"]["no_alert_for_skipped_runs"] = True r["parameters"].append({"name": "region", "default": "us-east-1"}) r["trigger"] = {"pause_status": "UNPAUSED", "table_update": {"table_names": ["samples.nyctaxi.trips"]}} +r["performance_target"] = "PERFORMANCE_OPTIMIZED" if "tags" not in r: r["tags"] = {} diff --git a/bundle/configsync/diff.go b/bundle/configsync/diff.go index 0d8bddc2e9..e8e347f094 100644 --- a/bundle/configsync/diff.go +++ b/bundle/configsync/diff.go @@ -85,13 +85,20 @@ func filterEntityDefaults(basePath string, value any) any { } func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDesc, error) { - hasConfigValue := cd.Old != nil || cd.New != nil + hasConfigValue := cd.New != nil normalizedValue, err := normalizeValue(cd.Remote) if err != nil { return nil, fmt.Errorf("failed to normalize remote value: %w", err) } if shouldSkipField(path, normalizedValue) { + // If the config has an explicit value for a server-side default field, + // we should remove it since the remote value is the default. + if cd.New != nil { + return &ConfigChangeDesc{ + Operation: OperationRemove, + }, nil + } return &ConfigChangeDesc{ Operation: OperationSkip, }, nil