diff --git a/Documentation/config/remote.adoc b/Documentation/config/remote.adoc index 91e46f66f5dd1c..9df8be27ebb157 100644 --- a/Documentation/config/remote.adoc +++ b/Documentation/config/remote.adoc @@ -107,6 +107,29 @@ priority configuration file (e.g. `.git/config` in a repository) to clear the values inherited from a lower priority configuration files (e.g. `$HOME/.gitconfig`). +remote..mustHave:: + When negotiating with this remote during `git fetch` and `git push`, + the client advertises a list of commits that exist locally. In + repos with many references, this list of "haves" can be truncated. + Depending on data shape, dropping certain references may be + expensive. This multi-valued config option specifies ref patterns + whose tips should always be sent as "have" commits during fetch + negotiation with this remote. ++ +Each value is either an exact ref name (e.g. `refs/heads/release`) or a +glob pattern (e.g. `refs/heads/release/*`). The pattern syntax is the same +as for `--negotiation-tip`. ++ +These config values are used as defaults for the `--must-have` command-line +option. If `--must-have` is specified on the command line, then the config +values are not used. ++ +This option is additive with the normal negotiation process: the +negotiation algorithm still runs and advertises its own selected commits, +but the refs matching `remote..mustHave` are sent unconditionally on +top of those heuristically selected commits. This option is also used +during push negotiation when `push.negotiate` is enabled. + remote..followRemoteHEAD:: How linkgit:git-fetch[1] should handle updates to `remotes//HEAD` when fetching using the configured refspecs of a remote. diff --git a/Documentation/fetch-options.adoc b/Documentation/fetch-options.adoc index 81a9d7f9bbc11d..fa3969d68d148b 100644 --- a/Documentation/fetch-options.adoc +++ b/Documentation/fetch-options.adoc @@ -69,6 +69,28 @@ See also the `fetch.negotiationAlgorithm` and `push.negotiate` configuration variables documented in linkgit:git-config[1], and the `--negotiate-only` option below. +`--must-have=`:: + Ensure that the given ref tip is always sent as a "have" line + during fetch negotiation, regardless of what the negotiation + algorithm selects. This is useful to guarantee that common + history reachable from specific refs is always considered, even + when `--negotiation-tip` restricts the set of tips or when the + negotiation algorithm would otherwise skip them. ++ +This option may be specified more than once; if so, each ref is sent +unconditionally. ++ +The argument may be an exact ref name (e.g. `refs/heads/release`) or a +glob pattern (e.g. `refs/heads/release/{asterisk}`). The pattern syntax +is the same as for `--negotiation-tip`. ++ +If `--negotiation-tip` is used, the have set is first restricted by that +option and then increased to include the tips specified by `--must-have`. ++ +If this option is not specified on the command line, then any +`remote..mustHave` config values for the current remote are used +instead. + `--negotiate-only`:: Do not fetch anything from the server, and instead print the ancestors of the provided `--negotiation-tip=` arguments, diff --git a/builtin/fetch.c b/builtin/fetch.c index 4795b2a13c30e3..fa491c106fd099 100644 --- a/builtin/fetch.c +++ b/builtin/fetch.c @@ -99,6 +99,7 @@ static struct transport *gsecondary; static struct refspec refmap = REFSPEC_INIT_FETCH; static struct string_list server_options = STRING_LIST_INIT_DUP; static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP; +static struct string_list must_have = STRING_LIST_INIT_NODUP; struct fetch_config { enum display_format display_format; @@ -1599,7 +1600,16 @@ static struct transport *prepare_transport(struct remote *remote, int deepen, if (transport->smart_options) add_negotiation_tips(transport->smart_options); else - warning("ignoring --negotiation-tip because the protocol does not support it"); + warning(_("ignoring %s because the protocol does not support it"), "--negotiation-tip"); + } + if (must_have.nr) { + if (transport->smart_options) + transport->smart_options->must_have = &must_have; + else + warning(_("ignoring %s because the protocol does not support it"), "--must-have"); + } else if (remote->must_have.nr) { + if (transport->smart_options) + transport->smart_options->must_have = &remote->must_have; } return transport; } @@ -2567,6 +2577,8 @@ int cmd_fetch(int argc, OPT_IPVERSION(&family), OPT_STRING_LIST(0, "negotiation-tip", &negotiation_tip, N_("revision"), N_("report that we have only objects reachable from this object")), + OPT_STRING_LIST(0, "must-have", &must_have, N_("revision"), + N_("ensure this ref is always sent as a negotiation have")), OPT_BOOL(0, "negotiate-only", &negotiate_only, N_("do not fetch a packfile; instead, print ancestors of negotiation tips")), OPT_PARSE_LIST_OBJECTS_FILTER(&filter_options), diff --git a/fetch-pack.c b/fetch-pack.c index 6ecd468ef766a8..afd4f31225f241 100644 --- a/fetch-pack.c +++ b/fetch-pack.c @@ -25,6 +25,7 @@ #include "oidset.h" #include "packfile.h" #include "odb.h" +#include "object-name.h" #include "path.h" #include "connected.h" #include "fetch-negotiator.h" @@ -332,6 +333,40 @@ static void send_filter(struct fetch_pack_args *args, } } +static int add_oid_to_oidset(const struct reference *ref, void *cb_data) +{ + struct oidset *set = cb_data; + oidset_insert(set, ref->oid); + return 0; +} + +static void resolve_must_have(const struct string_list *must_have, + struct oidset *result) +{ + struct string_list_item *item; + + if (!must_have || !must_have->nr) + return; + + for_each_string_list_item(item, must_have) { + if (!has_glob_specials(item->string)) { + struct object_id oid; + if (repo_get_oid(the_repository, item->string, &oid)) + continue; + if (!odb_has_object(the_repository->objects, &oid, 0)) + continue; + oidset_insert(result, &oid); + } else { + struct refs_for_each_ref_options opts = { + .pattern = item->string, + }; + refs_for_each_ref_ext( + get_main_ref_store(the_repository), + add_oid_to_oidset, result, &opts); + } + } +} + static int find_common(struct fetch_negotiator *negotiator, struct fetch_pack_args *args, int fd[2], struct object_id *result_oid, @@ -347,6 +382,7 @@ static int find_common(struct fetch_negotiator *negotiator, struct strbuf req_buf = STRBUF_INIT; size_t state_len = 0; struct packet_reader reader; + struct oidset must_have_oids = OIDSET_INIT; if (args->stateless_rpc && multi_ack == 1) die(_("the option '%s' requires '%s'"), "--stateless-rpc", "multi_ack_detailed"); @@ -474,7 +510,24 @@ static int find_common(struct fetch_negotiator *negotiator, trace2_region_enter("fetch-pack", "negotiation_v0_v1", the_repository); flushes = 0; retval = -1; + + /* Send unconditional haves from --must-have */ + resolve_must_have(args->must_have, &must_have_oids); + if (oidset_size(&must_have_oids)) { + struct oidset_iter iter; + oidset_iter_init(&must_have_oids, &iter); + + while ((oid = oidset_iter_next(&iter))) { + packet_buf_write(&req_buf, "have %s\n", + oid_to_hex(oid)); + print_verbose(args, "have %s", oid_to_hex(oid)); + } + } + while ((oid = negotiator->next(negotiator))) { + /* avoid duplicate oids from --must-have */ + if (oidset_contains(&must_have_oids, oid)) + continue; packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid)); print_verbose(args, "have %s", oid_to_hex(oid)); in_vain++; @@ -584,6 +637,7 @@ static int find_common(struct fetch_negotiator *negotiator, flushes++; } strbuf_release(&req_buf); + oidset_clear(&must_have_oids); if (!got_ready || !no_done) consume_shallow_list(args, &reader); @@ -1305,12 +1359,25 @@ static void add_common(struct strbuf *req_buf, struct oidset *common) static int add_haves(struct fetch_negotiator *negotiator, struct strbuf *req_buf, - int *haves_to_send) + int *haves_to_send, + struct oidset *must_have_oids) { int haves_added = 0; const struct object_id *oid; + /* Send unconditional haves from --must-have */ + if (must_have_oids) { + struct oidset_iter iter; + oidset_iter_init(must_have_oids, &iter); + + while ((oid = oidset_iter_next(&iter))) + packet_buf_write(req_buf, "have %s\n", + oid_to_hex(oid)); + } + while ((oid = negotiator->next(negotiator))) { + if (must_have_oids && oidset_contains(must_have_oids, oid)) + continue; packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid)); if (++haves_added >= *haves_to_send) break; @@ -1358,7 +1425,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, struct fetch_pack_args *args, const struct ref *wants, struct oidset *common, int *haves_to_send, int *in_vain, - int sideband_all, int seen_ack) + int sideband_all, int seen_ack, + struct oidset *must_have_oids) { int haves_added; int done_sent = 0; @@ -1413,7 +1481,8 @@ static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out, /* Add all of the common commits we've found in previous rounds */ add_common(&req_buf, common); - haves_added = add_haves(negotiator, &req_buf, haves_to_send); + haves_added = add_haves(negotiator, &req_buf, haves_to_send, + must_have_oids); *in_vain += haves_added; trace2_data_intmax("negotiation_v2", the_repository, "haves_added", haves_added); trace2_data_intmax("negotiation_v2", the_repository, "in_vain", *in_vain); @@ -1657,6 +1726,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, struct ref *ref = copy_ref_list(orig_ref); enum fetch_state state = FETCH_CHECK_LOCAL; struct oidset common = OIDSET_INIT; + struct oidset must_have_oids = OIDSET_INIT; struct packet_reader reader; int in_vain = 0, negotiation_started = 0; int negotiation_round = 0; @@ -1708,6 +1778,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, reader.me = "fetch-pack"; } + resolve_must_have(args->must_have, &must_have_oids); + while (state != FETCH_DONE) { switch (state) { case FETCH_CHECK_LOCAL: @@ -1747,7 +1819,8 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, &common, &haves_to_send, &in_vain, reader.use_sideband, - seen_ack)) { + seen_ack, + &must_have_oids)) { trace2_region_leave_printf("negotiation_v2", "round", the_repository, "%d", negotiation_round); @@ -1883,6 +1956,7 @@ static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args, negotiator->release(negotiator); oidset_clear(&common); + oidset_clear(&must_have_oids); return ref; } @@ -2181,12 +2255,14 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, const struct string_list *server_options, int stateless_rpc, int fd[], - struct oidset *acked_commits) + struct oidset *acked_commits, + const struct string_list *must_have) { struct fetch_negotiator negotiator; struct packet_reader reader; struct object_array nt_object_array = OBJECT_ARRAY_INIT; struct strbuf req_buf = STRBUF_INIT; + struct oidset must_have_oids = OIDSET_INIT; int haves_to_send = INITIAL_FLUSH; int in_vain = 0; int seen_ack = 0; @@ -2205,6 +2281,8 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, add_to_object_array, &nt_object_array); + resolve_must_have(must_have, &must_have_oids); + trace2_region_enter("fetch-pack", "negotiate_using_fetch", the_repository); while (!last_iteration) { int haves_added; @@ -2221,7 +2299,8 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, packet_buf_write(&req_buf, "wait-for-done"); - haves_added = add_haves(&negotiator, &req_buf, &haves_to_send); + haves_added = add_haves(&negotiator, &req_buf, &haves_to_send, + &must_have_oids); in_vain += haves_added; if (!haves_added || (seen_ack && in_vain >= MAX_IN_VAIN)) last_iteration = 1; @@ -2273,6 +2352,7 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, clear_common_flag(acked_commits); object_array_clear(&nt_object_array); + oidset_clear(&must_have_oids); negotiator.release(&negotiator); strbuf_release(&req_buf); } diff --git a/fetch-pack.h b/fetch-pack.h index 9d3470366f85ec..2e97ca5ea2b54f 100644 --- a/fetch-pack.h +++ b/fetch-pack.h @@ -23,6 +23,13 @@ struct fetch_pack_args { */ const struct oid_array *negotiation_tips; + /* + * If non-empty, ref patterns whose tips should always be sent + * as "have" lines during negotiation, regardless of what the + * negotiation algorithm selects. + */ + const struct string_list *must_have; + unsigned deepen_relative:1; unsigned quiet:1; unsigned keep_pack:1; @@ -93,7 +100,8 @@ void negotiate_using_fetch(const struct oid_array *negotiation_tips, const struct string_list *server_options, int stateless_rpc, int fd[], - struct oidset *acked_commits); + struct oidset *acked_commits, + const struct string_list *must_have); /* * Print an appropriate error message for each sought ref that wasn't diff --git a/remote.c b/remote.c index 7ca2a6501b4920..e07ec08fb3049f 100644 --- a/remote.c +++ b/remote.c @@ -152,6 +152,7 @@ static struct remote *make_remote(struct remote_state *remote_state, refspec_init_push(&ret->push); refspec_init_fetch(&ret->fetch); string_list_init_dup(&ret->server_options); + string_list_init_dup(&ret->must_have); ALLOC_GROW(remote_state->remotes, remote_state->remotes_nr + 1, remote_state->remotes_alloc); @@ -179,6 +180,7 @@ static void remote_clear(struct remote *remote) FREE_AND_NULL(remote->http_proxy); FREE_AND_NULL(remote->http_proxy_authmethod); string_list_clear(&remote->server_options, 0); + string_list_clear(&remote->must_have, 0); } static void add_merge(struct branch *branch, const char *name) @@ -562,6 +564,10 @@ static int handle_config(const char *key, const char *value, } else if (!strcmp(subkey, "serveroption")) { return parse_transport_option(key, value, &remote->server_options); + } else if (!strcmp(subkey, "musthave")) { + if (!value) + return config_error_nonbool(key); + string_list_append(&remote->must_have, value); } else if (!strcmp(subkey, "followremotehead")) { const char *no_warn_branch; if (!strcmp(value, "never")) diff --git a/remote.h b/remote.h index fc052945ee451d..e125313f45aa0b 100644 --- a/remote.h +++ b/remote.h @@ -117,6 +117,7 @@ struct remote { char *http_proxy_authmethod; struct string_list server_options; + struct string_list must_have; enum follow_remote_head_settings follow_remote_head; const char *no_warn_branch; diff --git a/send-pack.c b/send-pack.c index 67d6987b1ccd7e..baa52680bb3a9f 100644 --- a/send-pack.c +++ b/send-pack.c @@ -433,6 +433,7 @@ static void reject_invalid_nonce(const char *nonce, int len) static void get_commons_through_negotiation(struct repository *r, const char *url, + const struct string_list *must_have, const struct ref *remote_refs, struct oid_array *commons) { @@ -452,6 +453,14 @@ static void get_commons_through_negotiation(struct repository *r, nr_negotiation_tip++; } } + + if (must_have) { + struct string_list_item *item; + for_each_string_list_item(item, must_have) + strvec_pushf(&child.args, "--must-have=%s", + item->string); + } + strvec_push(&child.args, url); if (!nr_negotiation_tip) { @@ -528,7 +537,8 @@ int send_pack(struct repository *r, repo_config_get_bool(r, "push.negotiate", &push_negotiate); if (push_negotiate) { trace2_region_enter("send_pack", "push_negotiate", r); - get_commons_through_negotiation(r, args->url, remote_refs, &commons); + get_commons_through_negotiation(r, args->url, args->must_have, + remote_refs, &commons); trace2_region_leave("send_pack", "push_negotiate", r); } diff --git a/send-pack.h b/send-pack.h index c5ded2d2006f13..194a1898e54111 100644 --- a/send-pack.h +++ b/send-pack.h @@ -18,6 +18,7 @@ struct repository; struct send_pack_args { const char *url; + const struct string_list *must_have; unsigned verbose:1, quiet:1, porcelain:1, diff --git a/t/t5510-fetch.sh b/t/t5510-fetch.sh index 5dcb4b51a47d88..09e7b613a5c970 100755 --- a/t/t5510-fetch.sh +++ b/t/t5510-fetch.sh @@ -1728,6 +1728,127 @@ test_expect_success REFFILES "HEAD is updated even with conflicts" ' ) ' +test_expect_success '--must-have includes configured refs as haves' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # With --negotiation-tip restricting tips, only alpha_1 is + # normally sent. --must-have should also include beta_1. + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + --must-have=refs/tags/beta_1 \ + origin alpha_s beta_s && + + ALPHA_1=$(git -C client rev-parse alpha_1) && + test_grep "fetch> have $ALPHA_1" trace && + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace +' + +test_expect_success '--must-have works with glob patterns' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + --must-have="refs/tags/beta_*" \ + origin alpha_s beta_s && + + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace && + BETA_2=$(git -C client rev-parse beta_2) && + test_grep "fetch> have $BETA_2" trace +' + +test_expect_success '--must-have is additive with negotiation' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # Without --negotiation-tip, all local refs are used as tips. + # --must-have should add its refs unconditionally on top. + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --must-have=refs/tags/beta_1 \ + origin alpha_s beta_s && + + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace +' + +test_expect_success '--must-have ignores non-existent refs silently' ' + setup_negotiation_tip server server 0 && + + git -C client fetch --quiet \ + --negotiation-tip=alpha_1 \ + --must-have=refs/tags/nonexistent \ + origin alpha_s beta_s 2>err && + test_must_be_empty err +' + +test_expect_success '--must-have avoids duplicates with negotiator' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # Configure a ref that will also be a negotiation tip. + # fetch should still complete successfully. + ALPHA_1=$(git -C client rev-parse alpha_1) && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + --must-have=refs/tags/alpha_1 \ + origin alpha_s beta_s && + + # alpha_1 should appear as a have + test_grep "fetch> have $ALPHA_1" trace >matches && + test_line_count = 1 matches +' + +test_expect_success 'remote..mustHave used as default for --must-have' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # No --must-have on CLI; config should be used as default. + git -C client config --add remote.origin.mustHave refs/tags/beta_1 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + origin alpha_s beta_s && + + ALPHA_1=$(git -C client rev-parse alpha_1) && + test_grep "fetch> have $ALPHA_1" trace && + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace +' + +test_expect_success 'remote..mustHave works with glob patterns' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + git -C client config --add remote.origin.mustHave "refs/tags/beta_*" && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + origin alpha_s beta_s && + + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace && + BETA_2=$(git -C client rev-parse beta_2) && + test_grep "fetch> have $BETA_2" trace +' + +test_expect_success 'CLI --must-have overrides remote..mustHave' ' + test_when_finished rm -f trace && + setup_negotiation_tip server server 0 && + + # Config says beta_2, CLI says beta_1; only CLI should be used. + git -C client config --add remote.origin.mustHave refs/tags/beta_2 && + GIT_TRACE_PACKET="$(pwd)/trace" git -C client fetch \ + --negotiation-tip=alpha_1 \ + --must-have=refs/tags/beta_1 \ + origin alpha_s beta_s && + + BETA_1=$(git -C client rev-parse beta_1) && + test_grep "fetch> have $BETA_1" trace && + BETA_2=$(git -C client rev-parse beta_2) && + test_grep ! "fetch> have $BETA_2" trace +' + . "$TEST_DIRECTORY"/lib-httpd.sh start_httpd diff --git a/t/t5516-fetch-push.sh b/t/t5516-fetch-push.sh index 29e2f176081561..9272609eacff7d 100755 --- a/t/t5516-fetch-push.sh +++ b/t/t5516-fetch-push.sh @@ -254,6 +254,21 @@ test_expect_success 'push with negotiation does not attempt to fetch submodules' ! grep "Fetching submodule" err ' +test_expect_success 'push with negotiation and remote..mustHave' ' + test_when_finished rm -rf musthave && + mk_empty musthave && + git push musthave $the_first_commit:refs/remotes/origin/first_commit && + test_commit -C musthave unrelated_commit && + git -C musthave config receive.hideRefs refs/remotes/origin/first_commit && + test_when_finished "rm event" && + GIT_TRACE2_EVENT="$(pwd)/event" \ + git -c protocol.version=2 -c push.negotiate=1 \ + -c remote.musthave.mustHave=refs/heads/main \ + push musthave refs/heads/main:refs/remotes/origin/main && + test_grep \"key\":\"total_rounds\" event && + grep_wrote 2 event # 1 commit, 1 tree +' + test_expect_success 'push without wildcard' ' mk_empty testrepo && @@ -1349,7 +1364,7 @@ test_expect_success 'fetch follows tags by default' ' git for-each-ref >tmp1 && sed -n "p; s|refs/heads/main$|refs/remotes/origin/main|p" tmp1 | sed -n "p; s|refs/heads/main$|refs/remotes/origin/HEAD|p" | - sort -k 4 >../expect + sort -k 3 >../expect ) && test_when_finished "rm -rf dst" && git init dst && diff --git a/transport.c b/transport.c index 107f4fa5dce96a..e65f896ff372f7 100644 --- a/transport.c +++ b/transport.c @@ -464,6 +464,7 @@ static int fetch_refs_via_pack(struct transport *transport, args.stateless_rpc = transport->stateless_rpc; args.server_options = transport->server_options; args.negotiation_tips = data->options.negotiation_tips; + args.must_have = data->options.must_have; args.reject_shallow_remote = transport->smart_options->reject_shallow; if (!data->finished_handshake) { @@ -495,7 +496,8 @@ static int fetch_refs_via_pack(struct transport *transport, transport->server_options, transport->stateless_rpc, data->fd, - data->options.acked_commits); + data->options.acked_commits, + data->options.must_have); ret = 0; } goto cleanup; @@ -919,6 +921,7 @@ static int git_transport_push(struct transport *transport, struct ref *remote_re args.atomic = !!(flags & TRANSPORT_PUSH_ATOMIC); args.push_options = transport->push_options; args.url = transport->url; + args.must_have = &transport->remote->must_have; if (flags & TRANSPORT_PUSH_CERT_ALWAYS) args.push_cert = SEND_PACK_PUSH_CERT_ALWAYS; diff --git a/transport.h b/transport.h index 892f19454a75d6..7f8d779e9b408b 100644 --- a/transport.h +++ b/transport.h @@ -48,6 +48,12 @@ struct git_transport_options { */ struct oid_array *negotiation_tips; + /* + * If non-empty, ref patterns whose tips should always be sent + * as "have" lines during negotiation. + */ + const struct string_list *must_have; + /* * If allocated, whenever transport_fetch_refs() is called, add known * common commits to this oidset instead of fetching any packfiles.