Skip to content

send-pack: avoid sending the whole tree when pushing from a shallow clone #2208

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
newren wants to merge 1 commit into
base: ps/odb-pluggable-pack-generation
Choose a base branch
from
+260 −3
Open

send-pack: avoid sending the whole tree when pushing from a shallow clone #2208

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions Documentation/config/push.adoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,18 @@ This will result in only b (a and c are cleared).
rely solely on the server's ref advertisement to find commits
in common.

`push.shallowExcludeBoundary`::
When pushing from a shallow repository (see linkgit:git-clone[1]
`--depth`), Git normally assumes that the receiving end already
has the pushing repository's shallow grafts, and omits those
objects from the generated pack rather than resending the full
toplevel tree of those grafts. This is safe because the
receiving end rejects a push that references objects it does not
have. Set this to `false` to send those objects anyway; this is
only needed for the highly unusual case of using a push to seed
a receiver that adopts new shallow roots (i.e. a receiver that
has explicitly set `receive.shallowUpdate`). Default is `true`.

`push.useBitmaps`::
If set to `false`, disable use of bitmaps for `git push` even if
`pack.useBitmaps` is `true`, without preventing other git operations
Expand Down
95 changes: 95 additions & 0 deletions send-pack.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
#include "transport.h"
#include "version.h"
#include "oid-array.h"
#include "oidset.h"
#include "gpg-interface.h"
#include "shallow.h"
#include "parse-options.h"
Expand Down Expand Up @@ -55,6 +56,86 @@ static void append_negative_object(struct repository *r,
oid_array_append(haves, oid);
}

static int check_to_send_update(const struct ref *ref, const struct send_pack_args *args);

/*
* Add the shallow grafts (nr_parent == -1), which are reachable from the
* refs being pushed, to the pack boundary ("haves") as uninteresting
* (negative) tips so the generated pack leaves out everything beneath them.
*
* Walk only from the pushed tips, and only until a graft: using a graft
* that does not bound the pushed history could exclude an object we are
* genuinely sending (if it is also reachable from that unrelated graft).
* Stop early at any commit the peer already has, since it is a negative
* the peer can use and the graft beneath it would be redundant.
*/
static void append_reachable_shallow_grafts(struct repository *r,
struct ref *refs,
struct oid_array *advertised,
struct oid_array *negotiated,
struct send_pack_args *args,
struct oid_array *haves)
{
struct commit_list *pending = NULL;
struct oidset seen = OIDSET_INIT;
struct oidset known = OIDSET_INIT;
struct ref *ref;
size_t i;

for (i = 0; i < advertised->nr; i++)
oidset_insert(&known, &advertised->oid[i]);
for (i = 0; i < negotiated->nr; i++)
oidset_insert(&known, &negotiated->oid[i]);
for (ref = refs; ref; ref = ref->next)
if (!is_null_oid(&ref->old_oid))
oidset_insert(&known, &ref->old_oid);

for (ref = refs; ref; ref = ref->next) {
struct commit *commit;

if (is_null_oid(&ref->new_oid))
continue;
if (check_to_send_update(ref, args))
continue;
commit = lookup_commit_reference_gently(r, &ref->new_oid, 1);
if (commit)
commit_list_insert(commit, &pending);
}

while (pending) {
struct commit *commit = pop_commit(&pending);
const struct object_id *oid = &commit->object.oid;
struct commit_graft *graft;
struct commit_list *parent;

if (oidset_insert(&seen, oid))
continue;

/*
* A commit the peer already has bounds the pushed history
* with a negative it can use, so stop here rather than
* descend to a graft that would only be redundant.
*/
if (oidset_contains(&known, oid) &&
odb_has_object(r->objects, oid, 0))
continue;

graft = lookup_commit_graft(r, oid);
if (graft && graft->nr_parent == -1) {
append_negative_object(r, haves, oid);
continue;
}

if (repo_parse_commit(r, commit))
continue;
for (parent = commit->parents; parent; parent = parent->next)
commit_list_insert(parent->item, &pending);
}

oidset_clear(&seen);
oidset_clear(&known);
}

/*
* Make a pack stream and spit it out into file descriptor fd
*/
Expand Down Expand Up @@ -88,6 +169,20 @@ static int pack_objects(struct repository *r,
for (size_t i = 0; i < negotiated->nr; i++)
append_negative_object(r, &opts.haves, &negotiated->oid[i]);

/*
* When pushing from a shallow repository, avoid re-pushing the
* entire toplevel tree.
*/
if (is_repository_shallow(r)) {
int exclude_boundary = 1;
repo_config_get_bool(r, "push.shallowexcludeboundary",
&exclude_boundary);
if (exclude_boundary)
append_reachable_shallow_grafts(r, refs, advertised,
negotiated, args,
&opts.haves);
}

while (refs) {
if (!is_null_oid(&refs->old_oid))
append_negative_object(r, &opts.haves, &refs->old_oid);
Expand Down
156 changes: 153 additions & 3 deletions t/t5538-push-shallow.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,8 @@ EOF
test_expect_success 'push from shallow clone, with grafted roots' '
(
cd shallow2 &&
test_must_fail git push ../.git +main:refs/remotes/shallow2/main 2>err &&
test_must_fail git -c push.shallowExcludeBoundary=false \
push ../.git +main:refs/remotes/shallow2/main 2>err &&
test_grep "shallow2/main.*shallow update not allowed" err
) &&
test_must_fail git rev-parse shallow2/main &&
Expand All @@ -75,7 +76,8 @@ test_expect_success 'add new shallow root with receive.updateshallow on' '
test_config receive.shallowupdate true &&
(
cd shallow2 &&
git push ../.git +main:refs/remotes/shallow2/main
git -c push.shallowExcludeBoundary=false \
push ../.git +main:refs/remotes/shallow2/main
) &&
git log --format=%s shallow2/main >actual &&
git fsck &&
Expand All @@ -90,7 +92,8 @@ test_expect_success 'push from shallow to shallow' '
(
cd shallow &&
git --git-dir=../shallow2/.git config receive.shallowupdate true &&
git push ../shallow2/.git +main:refs/remotes/shallow/main &&
git -c push.shallowExcludeBoundary=false \
push ../shallow2/.git +main:refs/remotes/shallow/main &&
git --git-dir=../shallow2/.git config receive.shallowupdate false
) &&
(
Expand Down Expand Up @@ -164,4 +167,151 @@ test_expect_success 'push new commit from shallow clone has good deltas' '
test_region pack-objects path-walk config-push.txt
'

test_expect_success 'shallow push only pushes what is necessary' '
git init adv-origin &&
# The shallow grafts are intentionally untagged so that no
# advertised ref points at them.
test_commit --no-tag -C adv-origin a &&
test_commit --no-tag -C adv-origin b &&

git clone --depth=1 "file://$(pwd)/adv-origin" adv-client &&

# The remote branch advances past the history we have, so its
# advertised tip is something we cannot use as a negative tip;
# only the shallow graft lets us exclude the full tree.
test_commit --no-tag -C adv-origin c &&

git -C adv-client checkout -b topic &&
test_commit --no-tag -C adv-client new &&
GIT_PROGRESS_DELAY=0 git -C adv-client push --progress origin topic 2>err &&

# Only the new commit, its tree, and the new blob are sent; sending
# the full tree is avoided by excluding the shallow graft.
test_grep "Enumerating objects: 4, done." err
'

test_expect_success 'push.shallowExcludeBoundary=false sends full tree' '
git init adv-origin2 &&
test_commit --no-tag -C adv-origin2 a &&
test_commit --no-tag -C adv-origin2 b &&

git clone --depth=1 "file://$(pwd)/adv-origin2" adv-client2 &&
test_commit --no-tag -C adv-origin2 c &&

git -C adv-client2 checkout -b topic &&
test_commit --no-tag -C adv-client2 new &&
GIT_PROGRESS_DELAY=0 git -C adv-client2 \
-c push.shallowExcludeBoundary=false \
push --progress origin topic 2>err &&

# With the optimization disabled and no advertised ref pointing at
# the shallow graft, the full snapshot down to the shallow graft is
# resent, including its full tree.
test_grep "Enumerating objects: 7, done." err
'

# A rejected ref must not over-exclude objects that another, accepted ref
# legitimately needs in the pack. Set up a testcase using two independent
# shallow roots.
#
# origin: two unrelated histories; only branch A carries blob O (sh=shared)
# A: A0---A1 (A0, A1 trees contain sh=O)
# B: B0---B1 (no "shared" blob)
#
# receiver: seeded from branch B only, under both ref names; lacks blob O
# refs/heads/B -> B1
# refs/heads/A -> B1 (makes our A push a non-fast-forward)
#
# client: "clone --depth=1 --no-single-branch" gives a graft at each tip
# and a copy of blob O under A1 (x = cut parents = shallow graft)
# x x
# | |
# A1 B1
# | |
# cX topic=cY (cY re-adds sh=O, which the receiver lacks)
#
# push "A topic" (non-atomic):
# A -> a non-fast-forward vs receiver A=B1, so its ref update is
# rejected locally and never applied. It still takes part in
# the shared pack computation, and the buggy code also walked
# back from it to graft A1 (which owns O).
# topic -> accepted; cY grafts onto B1 and needs blob O.
#
# Using the shallow graft A1 (an ancestor of A) to trim the pack, even
# though our push of A is rejected locally, would omit blob O from topic's
# pack -- yet topic needs O. We want to ensure that when topic is pushed,
# O is sent along with it despite A being rejected.
test_expect_success 'shallow push does not over-exclude for an accepted ref via a rejected one' '
# origin
git init tworoot-origin &&
git -C tworoot-origin checkout -b A &&
test_commit -C tworoot-origin --no-tag has-shared sh shared &&
test_commit -C tworoot-origin --no-tag A1 &&
git -C tworoot-origin switch --orphan B &&
test_commit -C tworoot-origin --no-tag B0 &&
test_commit -C tworoot-origin --no-tag B1 &&

# receiver: branch B only, exposed as both B and A
git init --bare tworoot-receiver.git &&
git -C tworoot-origin push "file://$(pwd)/tworoot-receiver.git" \
B:refs/heads/B B:refs/heads/A &&

# client: a shallow graft at each branch tip
git clone --depth=1 --no-single-branch \
"file://$(pwd)/tworoot-origin" tworoot-client &&

# branch A gets commit cX; including A in the push gives us a
# locally-rejected ref whose graft A1 the buggy code walked to. The A
# ref update is a non-fast-forward, so it is rejected and never applied.
git -C tworoot-client checkout A &&
test_commit -C tworoot-client --no-tag cX &&

# branch topic is what we actually send, reintroducing blob O on B1
git -C tworoot-client checkout -b topic B &&
test_commit -C tworoot-client --no-tag reintroduce sh shared &&

# push both in one command: they share a single pack computation, so a
# graft reached from the rejected A can strip objects that topic needs.
# The A ref update is rejected locally (non-fast-forward); the shared
# pack must still contain blob O for topic to land on the receiver.
test_must_fail git -C tworoot-client push \
"file://$(pwd)/tworoot-receiver.git" A topic &&
git --git-dir=tworoot-receiver.git rev-parse --verify topic
'

# push.shallowExcludeBoundary (default true) omits the shallow boundary
# snapshot from the pack, since an ordinary receiver already has it. The
# exception is a receiver willing to adopt a *new* shallow root
# (receive.shallowUpdate): it genuinely needs that snapshot, so the default
# optimization leaves it unable to graft the new root. Verify the receiver
# rejects such a push (rather than corrupting itself), and that setting the
# config to false restores the full snapshot and lets the push succeed. This
# is the tradeoff that motivates the config knob.
test_expect_success 'default push to a shallowUpdate receiver rejects a rootless snapshot' '
git init seed-origin &&
test_commit -C seed-origin s1 &&
test_commit -C seed-origin s2 &&
test_commit -C seed-origin s3 &&

# depth-2: a shallow graft at s2, pushing s3 on top of it
git clone --depth=2 "file://$(pwd)/seed-origin" seed-client &&

git init --bare seed-receiver.git &&
git --git-dir=seed-receiver.git config receive.shallowUpdate true &&

# Default (optimization on): the s2 boundary snapshot is withheld, so
# the receiver cannot graft the new root and rejects the push, leaving
# the ref uncreated.
test_must_fail git -C seed-client push \
"file://$(pwd)/seed-receiver.git" HEAD:refs/heads/seeded 2>err &&
test_grep "remote rejected" err &&
test_must_fail git --git-dir=seed-receiver.git rev-parse --verify seeded &&

# Opt-out: the full snapshot is sent, so the same push now succeeds and
# the new shallow root is grafted.
git -C seed-client -c push.shallowExcludeBoundary=false push \
"file://$(pwd)/seed-receiver.git" HEAD:refs/heads/seeded &&
git --git-dir=seed-receiver.git rev-parse --verify seeded
'

test_done
Loading