Ilya kuznetsov data/config sync performance optimized

acceptance/bundle/config-remote-sync/config_edits/databricks.yml.tmpl

@@ -34,6 +34,7 @@ targets:
             env: dev
             version: v1
             team: data-team
+          description: "original description"
           max_concurrent_runs: 1
           environments:
             - environment_key: default

acceptance/bundle/config-remote-sync/config_edits/output.txt

@@ -14,18 +14,22 @@ Deployment complete!
 
 === Case 5: Updated locally, removed remotely
 
-=== Case 6: Added locally and remotely with same value (no drift expected)
+=== Case 6: Removed locally, edited remotely
+
+=== Case 7: Added locally and remotely with same value (no drift expected)
 
 === Edit job remotely
 
 === Detect and save changes
 Detected changes in 1 resource(s):
 
 Resource: resources.jobs.my_job
+  description: add
   email_notifications.on_failure: replace
   email_notifications.on_failure[0]: replace
   max_concurrent_runs: replace
   tags['env']: remove
+  timeout_seconds: remove
 
 
 
@@ -34,22 +38,30 @@ Resource: resources.jobs.my_job
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -24,5 +24,5 @@
+@@ -13,4 +13,5 @@
+             node_type_id: [NODE_TYPE_ID]
+             num_workers: 1
++      description: updated remotely
+
+ targets:
+@@ -23,5 +24,5 @@
                - success@example.com
              on_failure:
 -              - config-failure@example.com
 +              - remote-failure@example.com
            parameters:
              - name: catalog
-@@ -35,7 +35,6 @@
+@@ -34,9 +35,7 @@
                unit: DAYS
            tags:
 -            env: config-production
              team: data-team
+             project: analytics
 -          max_concurrent_runs: 3
+-          timeout_seconds: 3600
 +          max_concurrent_runs: 5
-           timeout_seconds: 3600
            environments:
+             - environment_key: default
 
 >>> [CLI] bundle destroy --auto-approve
 The following resources will be deleted:

acceptance/bundle/config-remote-sync/config_edits/script

@@ -82,22 +82,38 @@ read -r -d '' case5 <<'EOF' || true
 r["tags"].pop("env", None)
 EOF
 
-title "Case 6: Added locally and remotely with same value (no drift expected)"
+title "Case 6: Removed locally, edited remotely"
 echo
 old=$(cat <<'EOF'
-        my_job:
-          email_notifications:
+          description: "original description"
+          max_concurrent_runs: 3
 EOF
 )
 new=$(cat <<'EOF'
-        my_job:
-          description: A test job
-          email_notifications:
+          max_concurrent_runs: 3
 EOF
 )
 update_file.py databricks.yml "$old" "$new"
 read -r -d '' case6 <<'EOF' || true
-r["description"] = "A test job"
+r["description"] = "updated remotely"
+EOF
+
+title "Case 7: Added locally and remotely with same value (no drift expected)"
+echo
+old=$(cat <<'EOF'
+            team: data-team
+EOF
+)
+new=$(cat <<'EOF'
+            team: data-team
+            project: analytics
+EOF
+)
+update_file.py databricks.yml "$old" "$new"
+read -r -d '' case7 <<'EOF' || true
+if "tags" not in r:
+    r["tags"] = {}
+r["tags"]["project"] = "analytics"
 EOF
 
 title "Edit job remotely"
@@ -114,6 +130,8 @@ $case4
 $case5
 
 $case6
+
+$case7
 EOF
 
 title "Detect and save changes"

acceptance/bundle/config-remote-sync/job_fields/databricks.yml.tmpl

@@ -4,6 +4,7 @@ bundle:
 resources:
   jobs:
     my_job:
+      performance_target: STANDARD
       email_notifications:
         on_success:
           - success@example.com

acceptance/bundle/config-remote-sync/job_fields/output.txt

@@ -13,6 +13,7 @@ Resource: resources.jobs.my_job
   email_notifications.no_alert_for_skipped_runs: add
   email_notifications.on_failure: add
   parameters: replace
+  performance_target: remove
   tags['team']: add
   trigger.pause_status: add
   trigger.periodic: remove
@@ -25,7 +26,11 @@ Resource: resources.jobs.my_job
 >>> diff.py databricks.yml.backup databricks.yml
 --- databricks.yml.backup
 +++ databricks.yml
-@@ -8,13 +8,19 @@
+@@ -5,17 +5,22 @@
+   jobs:
+     my_job:
+-      performance_target: STANDARD
+       email_notifications:
          on_success:
            - success@example.com
 +        no_alert_for_skipped_runs: true
@@ -52,7 +57,7 @@ Resource: resources.jobs.my_job
 +            - samples.nyctaxi.trips
        environments:
          - environment_key: default
-@@ -31,4 +37,6 @@
+@@ -32,4 +37,6 @@
              node_type_id: [NODE_TYPE_ID]
              num_workers: 1
 +      tags:

acceptance/bundle/config-remote-sync/job_fields/script

@@ -19,6 +19,7 @@ r["email_notifications"]["on_failure"] = ["failure@example.com"]
 r["email_notifications"]["no_alert_for_skipped_runs"] = True
 r["parameters"].append({"name": "region", "default": "us-east-1"})
 r["trigger"] = {"pause_status": "UNPAUSED", "table_update": {"table_names": ["samples.nyctaxi.trips"]}}
+r["performance_target"] = "PERFORMANCE_OPTIMIZED"
 
 if "tags" not in r:
     r["tags"] = {}

bundle/configsync/diff.go

@@ -85,13 +85,20 @@ func filterEntityDefaults(basePath string, value any) any {
 }
 
 func convertChangeDesc(path string, cd *deployplan.ChangeDesc) (*ConfigChangeDesc, error) {
-	hasConfigValue := cd.Old != nil || cd.New != nil
+	hasConfigValue := cd.New != nil
 	normalizedValue, err := normalizeValue(cd.Remote)
 	if err != nil {
 		return nil, fmt.Errorf("failed to normalize remote value: %w", err)
 	}
 
 	if shouldSkipField(path, normalizedValue) {
+		// If the config has an explicit value for a server-side default field,
+		// we should remove it since the remote value is the default.
+		if cd.New != nil {
+			return &ConfigChangeDesc{
+				Operation: OperationRemove,
+			}, nil
+		}
 		return &ConfigChangeDesc{
 			Operation: OperationSkip,
 		}, nil