From 63f25a0762348c7ed63a557f0ef214c3c09e6ec8 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 23 Jan 2026 11:35:05 -0500 Subject: [PATCH 1/2] scalar: add ---cache-server-url options In #836, we implemented custom gvfs..cache-server config options that allow existing enlistments to point specific GVFS Protocol verbs at different cache servers. However, that did not help at clone time. This change adds ---cache-server-url options to 'scalar clone' to allow initial clones to take advantage of different cache servers. The intended use here is to take advantage of new cache server infrastructure as each protocol endpoint is deployed independently. Signed-off-by: Derrick Stolee --- Documentation/scalar.adoc | 9 +++++++-- scalar.c | 40 +++++++++++++++++++++++++++++++++++++- t/t9210-scalar.sh | 41 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 87 insertions(+), 3 deletions(-) diff --git a/Documentation/scalar.adoc b/Documentation/scalar.adoc index 56c99c8a2d00dc..e08f22607f0267 100644 --- a/Documentation/scalar.adoc +++ b/Documentation/scalar.adoc @@ -10,8 +10,8 @@ SYNOPSIS [verse] scalar clone [--single-branch] [--branch ] [--full-clone] [--[no-]src] [--[no-]tags] [--[no-]maintenance] - [--[no-]src] [--local-cache-path ] [--cache-server-url ] - [] + [--cache-server-url ] [--[verb]-cache-server-url ] + [--local-cache-path ] [] scalar list scalar register [--[no-]maintenance] [] scalar unregister [] @@ -121,6 +121,11 @@ clone), and `~/.scalarCache` on macOS. Retrieve missing objects from the specified remote, which is expected to understand the GVFS protocol. +--[verb]-cache-server-url :: + Set the appropriate `gvfs..cache-server` config value that overrides + the provided `--cache-server-url` or the dynamically discovered URL. The + list of allowed verbs is `prefetch`, `get`, and `post`. + --gvfs-protocol:: --no-gvfs-protocol:: When cloning from a `` with either `dev.azure.com` or diff --git a/scalar.c b/scalar.c index 7dcca36a86a249..0e73108f9d598a 100644 --- a/scalar.c +++ b/scalar.c @@ -783,6 +783,7 @@ static int cmd_clone(int argc, const char **argv) int src = 1, tags = 1, maintenance = 1; const char *cache_server_url = NULL, *local_cache_root = NULL; char *default_cache_server_url = NULL, *local_cache_root_abs = NULL; + const char *prefetch_server = NULL, *get_server = NULL, *post_server = NULL; int gvfs_protocol = -1; const char *ref_format = NULL; @@ -805,6 +806,15 @@ static int cmd_clone(int argc, const char **argv) OPT_STRING(0, "cache-server-url", &cache_server_url, N_(""), N_("the url or friendly name of the cache server")), + OPT_STRING(0, "prefetch-cache-server-url", &prefetch_server, + N_(""), + N_("the url or friendly name of a cache server for the prefetch endpoint")), + OPT_STRING(0, "get-cache-server-url", &get_server, + N_(""), + N_("the url or friendly name of a cache server for the objects GET endpoint")), + OPT_STRING(0, "post-cache-server-url", &post_server, + N_(""), + N_("the url or friendly name of a cache server for the objects POST endpoint")), OPT_STRING(0, "local-cache-path", &local_cache_root, N_(""), N_("override the path for the local Scalar cache")), @@ -817,7 +827,8 @@ static int cmd_clone(int argc, const char **argv) const char * const clone_usage[] = { N_("scalar clone [--single-branch] [--branch ] [--full-clone]\n" "\t[--[no-]src] [--[no-]tags] [--[no-]maintenance] [--ref-format ]\n" - "\t []"), + "\t[--cache-server-url ] [--[verb]-cache-server-url ]\n" + "\t[--local-cache-path ] []"), NULL }; const char *url; @@ -979,6 +990,33 @@ static int cmd_clone(int argc, const char **argv) if (cache_server_url) fprintf(stderr, "Cache server URL: %s\n", cache_server_url); + + if (prefetch_server && + set_config("gvfs.prefetch.cache-server=%s", prefetch_server)) { + res = error(_("could not configure prefetch cache server")); + goto cleanup; + } + if (prefetch_server) + fprintf(stderr, "Prefetch cache server URL: %s\n", + prefetch_server); + + if (get_server && + set_config("gvfs.get.cache-server=%s", get_server)) { + res = error(_("could not configure objects GET cache server")); + goto cleanup; + } + if (get_server) + fprintf(stderr, "Objects GET cache server URL: %s\n", + get_server); + + if (post_server && + set_config("gvfs.post.cache-server=%s", post_server)) { + res = error(_("could not configure objects POST cache server")); + goto cleanup; + } + if (post_server) + fprintf(stderr, "Objects POST cache server URL: %s\n", + post_server); } else { if (set_config("core.useGVFSHelper=false") || set_config("remote.origin.promisor=true") || diff --git a/t/t9210-scalar.sh b/t/t9210-scalar.sh index 493a35b8f43a5f..322b8b8b39461d 100755 --- a/t/t9210-scalar.sh +++ b/t/t9210-scalar.sh @@ -455,6 +455,47 @@ test_expect_success '`scalar clone` with GVFS-enabled server' ' ) ' +. "$TEST_DIRECTORY"/lib-gvfs-helper.sh + +test_expect_success 'scalar clone: all verbs with different servers' ' + git config --global core.askPass true && + + test_when_finished "per_test_cleanup" && + test_when_finished "scalar delete scalar-clone" && + + start_gvfs_protocol_server 1 && + start_gvfs_protocol_server 2 && + start_gvfs_protocol_server 3 && + + # Configure each verb to use a different server: + # - server 0: default (unused in this test; not running.) + # - server 1: prefetch + # - server 2: get + # - server 3: post + scalar -c credential.interactive=true \ + clone --full-clone \ + --cache-server-url="$(cache_server_url 0)" \ + --prefetch-cache-server-url="$(cache_server_url 1)" \ + --get-cache-server-url="$(cache_server_url 2)" \ + --post-cache-server-url="$(cache_server_url 3)" \ + --gvfs-protocol \ + -- "http://$HOST_PORT/" scalar-clone 2>err >out && + + test_grep "Cache server URL: $(cache_server_url 0)" err && + test_grep "Prefetch cache server URL: $(cache_server_url 1)" err && + test_grep "Objects GET cache server URL: $(cache_server_url 2)" err && + test_grep "Objects POST cache server URL: $(cache_server_url 3)" err && + + test_cmp_config -C scalar-clone/src "$(cache_server_url 0)" gvfs.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 1)" gvfs.prefetch.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 2)" gvfs.get.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 3)" gvfs.post.cache-server && + + verify_server_was_contacted 1 && + verify_server_was_contacted 2 && + verify_server_was_contacted 3 +' + test_expect_success 'fetch does not hang in gvfs-helper' ' test_must_fail git -C using-gvfs/src fetch origin does-not-exist ' From 3c035eedd3914b47d61482d8f4c134b13619dbb6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 28 Jan 2026 10:09:16 -0500 Subject: [PATCH 2/2] t9210: differentiate origin and cache servers Now that we are testing the ---cache-server options in t9210, we need to be more careful about our variable names to ensure those created in t9210-scalar.sh are different from those set in lib-gvfs-helper.sh. For this we: * Rename variables to use ORIGIN_* for those that host Git repo data. * Update server output files to have more specific names. * Avoid using the default test server port when possible, in preference for a positive shift-by-10k. Signed-off-by: Derrick Stolee --- t/lib-gvfs-helper.sh | 10 +++--- t/t9210-scalar.sh | 78 +++++++++++++++++++++++--------------------- 2 files changed, 46 insertions(+), 42 deletions(-) diff --git a/t/lib-gvfs-helper.sh b/t/lib-gvfs-helper.sh index 32c951361e3282..3f84d44ef0ba22 100644 --- a/t/lib-gvfs-helper.sh +++ b/t/lib-gvfs-helper.sh @@ -58,8 +58,8 @@ SHARED_CACHE_T2="$(pwd)"/shared_cache_t2 # The server will shut down if/when we delete it. (This is a little # easier than killing it by PID.) # -PID_FILE="$(pwd)"/pid-file.pid -SERVER_LOG="$(pwd)"/OUT.server.log +PID_FILE="$(pwd)"/pid-file-gvfs.pid +SERVER_LOG="$(pwd)"/OUT.gvfs.server.log # Helper functions to compute port, pid-file, and log for a given # port increment. An increment of 0 (or empty) uses the base values. @@ -294,7 +294,7 @@ stop_gvfs_protocol_server () { # port before the next test start another instance and it attempts to # bind to it). # - for k in 0 1 2 3 4 + for k in $(test_seq 5) do if grep -q "Starting graceful shutdown" "$log_file" then @@ -334,7 +334,7 @@ start_gvfs_protocol_server () { # # Give it a few seconds to get started. # - for k in 0 1 2 3 4 + for k in $(test_seq 5) do if test -f "$pid_file" then @@ -375,7 +375,7 @@ start_gvfs_protocol_server_with_mayhem () { # # Give it a few seconds to get started. # - for k in 0 1 2 3 4 + for k in $(test_seq 5) do if test -f "$PID_FILE" then diff --git a/t/t9210-scalar.sh b/t/t9210-scalar.sh index 322b8b8b39461d..588a09355b9286 100755 --- a/t/t9210-scalar.sh +++ b/t/t9210-scalar.sh @@ -360,20 +360,20 @@ test_expect_success UNZIP 'scalar diagnose' ' GIT_TEST_ALLOW_GVFS_VIA_HTTP=1 export GIT_TEST_ALLOW_GVFS_VIA_HTTP -test_set_port GIT_TEST_GVFS_PROTOCOL_PORT -HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_PORT -PID_FILE="$(pwd)"/pid-file.pid -SERVER_LOG="$(pwd)"/OUT.server.log +test_set_port GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT +ORIGIN_HOST_PORT=127.0.0.1:$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT +ORIGIN_PID_FILE="$(pwd)"/pid-file.$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT.pid +ORIGIN_SERVER_LOG="$(pwd)"/OUT.server.$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT.log test_atexit ' - test -f "$PID_FILE" || return 0 + test -f "$ORIGIN_PID_FILE" || return 0 # The server will shutdown automatically when we delete the pid-file. - rm -f "$PID_FILE" + rm -f "$ORIGIN_PID_FILE" test -z "$verbose$verbose_log" || { echo "server log:" - cat "$SERVER_LOG" + cat "$ORIGIN_SERVER_LOG" } # Give it a few seconds to shutdown (mainly to completely release the @@ -381,7 +381,7 @@ test_atexit ' # bind to it). for k in $(test_seq 5) do - grep -q "Starting graceful shutdown" "$SERVER_LOG" && + grep -q "Starting graceful shutdown" "$ORIGIN_SERVER_LOG" && return 0 || sleep 1 done @@ -394,14 +394,14 @@ start_gvfs_enabled_http_server () { GIT_HTTP_EXPORT_ALL=1 \ test-gvfs-protocol --verbose \ --listen=127.0.0.1 \ - --port=$GIT_TEST_GVFS_PROTOCOL_PORT \ + --port=$GIT_TEST_GVFS_PROTOCOL_ORIGIN_PORT \ --reuseaddr \ - --pid-file="$PID_FILE" \ - 2>"$SERVER_LOG" & + --pid-file="$ORIGIN_PID_FILE" \ + 2>"$ORIGIN_SERVER_LOG" & - for k in 0 1 2 3 4 + for k in $(test_seq 5) do - if test -f "$PID_FILE" + if test -f "$ORIGIN_PID_FILE" then return 0 fi @@ -426,12 +426,12 @@ test_expect_success '`scalar clone` with GVFS-enabled server' ' GIT_TRACE2_EVENT="$(pwd)/clone-trace-with-gvfs" scalar \ -c credential.interactive=true \ clone --gvfs-protocol \ - --single-branch -- http://$HOST_PORT/ using-gvfs && + --single-branch -- http://$ORIGIN_HOST_PORT/ using-gvfs && grep "GET/config(main)" expect && @@ -439,7 +439,7 @@ test_expect_success '`scalar clone` with GVFS-enabled server' ' test_cmp expect actual && : verify that URL-specific HTTP version setting is configured for GVFS URLs in clone && - git -C using-gvfs/src config "http.http://$HOST_PORT/.version" >actual && + git -C using-gvfs/src config "http.http://$ORIGIN_HOST_PORT/.version" >actual && echo "HTTP/1.1" >expect && test_cmp expect actual && @@ -466,37 +466,41 @@ test_expect_success 'scalar clone: all verbs with different servers' ' start_gvfs_protocol_server 1 && start_gvfs_protocol_server 2 && start_gvfs_protocol_server 3 && + start_gvfs_protocol_server 4 && # Configure each verb to use a different server: - # - server 0: default (unused in this test; not running.) - # - server 1: prefetch - # - server 2: get - # - server 3: post + # - server 1: default (unused in this test; not running.) + # - server 2: prefetch + # - server 3: get + # - server 4: post scalar -c credential.interactive=true \ clone --full-clone \ - --cache-server-url="$(cache_server_url 0)" \ - --prefetch-cache-server-url="$(cache_server_url 1)" \ - --get-cache-server-url="$(cache_server_url 2)" \ - --post-cache-server-url="$(cache_server_url 3)" \ + --cache-server-url="$(cache_server_url 1)" \ + --prefetch-cache-server-url="$(cache_server_url 2)" \ + --get-cache-server-url="$(cache_server_url 3)" \ + --post-cache-server-url="$(cache_server_url 4)" \ --gvfs-protocol \ - -- "http://$HOST_PORT/" scalar-clone 2>err >out && + -- "http://$ORIGIN_HOST_PORT/" scalar-clone 2>err >out && - test_grep "Cache server URL: $(cache_server_url 0)" err && - test_grep "Prefetch cache server URL: $(cache_server_url 1)" err && - test_grep "Objects GET cache server URL: $(cache_server_url 2)" err && - test_grep "Objects POST cache server URL: $(cache_server_url 3)" err && + test_grep "Cache server URL: $(cache_server_url 1)" err && + test_grep "Prefetch cache server URL: $(cache_server_url 2)" err && + test_grep "Objects GET cache server URL: $(cache_server_url 3)" err && + test_grep "Objects POST cache server URL: $(cache_server_url 4)" err && - test_cmp_config -C scalar-clone/src "$(cache_server_url 0)" gvfs.cache-server && - test_cmp_config -C scalar-clone/src "$(cache_server_url 1)" gvfs.prefetch.cache-server && - test_cmp_config -C scalar-clone/src "$(cache_server_url 2)" gvfs.get.cache-server && - test_cmp_config -C scalar-clone/src "$(cache_server_url 3)" gvfs.post.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 1)" gvfs.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 2)" gvfs.prefetch.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 3)" gvfs.get.cache-server && + test_cmp_config -C scalar-clone/src "$(cache_server_url 4)" gvfs.post.cache-server && verify_server_was_contacted 1 && verify_server_was_contacted 2 && verify_server_was_contacted 3 ' -test_expect_success 'fetch does not hang in gvfs-helper' ' +test_expect_success EXPENSIVE 'fetch does not hang in gvfs-helper' ' + # Marked as EXPENSIVE as this will go through multiple rounds of + # exponential backoff, including delays of 8, 16, 32, 64, 128, + # and 256 seconds in two separate instances. test_must_fail git -C using-gvfs/src fetch origin does-not-exist ' @@ -510,7 +514,7 @@ test_expect_success '`scalar clone --no-gvfs-protocol` skips gvfs/config' ' GIT_TRACE2_EVENT="$(pwd)/clone-trace-no-gvfs" scalar \ -c credential.interactive=true \ clone --no-gvfs-protocol \ - --single-branch -- http://$HOST_PORT/ skipping-gvfs && + --single-branch -- http://$ORIGIN_HOST_PORT/ skipping-gvfs && ! grep "GET/config(main)" out && cat >expect <<-EOF && - #0: http://$HOST_PORT/servertype/cache + #0: http://$ORIGIN_HOST_PORT/servertype/cache EOF test_cmp expect out &&