connected: add incremental connectivity check

Documentation/config/transfer.adoc

@@ -1,3 +1,23 @@
+transfer.connectivityCheck::
+	Choose which algorithm to use for the connectivity check
+	performed during object transfer operations such as
+	linkgit:git-fetch[1] and linkgit:git-receive-pack[1].
+	The connectivity check verifies that all objects reachable
+	from the incoming tips are available locally or, in a partial
+	clone, promised by a promisor remote.
+	The variants are as follows:
++
+--
+`full` (default);;
+	Walk the full object closure of the boundary commits.
+`incremental`;;
+	Verify incoming commits by diffing their trees against parent
+	trees, recursively descending only into entries that differ.
+	The largest benefits occur when incoming commits change a
+	small fraction of a large tree closure.
+	Falls back to `full` for deepening fetches.
+--
+
 transfer.credentialsInUrl::
 	A configured URL can contain plaintext credentials in the form
 	`<protocol>://<user>:<password>@<domain>/<path>`. You may want

Documentation/rev-list-options.adoc

@@ -1089,6 +1089,12 @@ we cannot get their Object ID though, an error will be raised.
 	stronger than `--missing=allow-promisor` because it limits the
 	traversal, rather than just silencing errors about missing
 	objects.
+
+`--verify-trees-incremental`::
+	(For internal use only.)  Verify tree connectivity
+	incrementally by comparing each commit's tree against its
+	parent trees.  Used by `check_connected()` when
+	`transfer.connectivityCheck` is set to `incremental`.
 endif::git-rev-list[]
 
 `--no-walk[=(sorted|unsorted)]`::

Makefile

@@ -812,6 +812,7 @@ TEST_BUILTINS_OBJS += test-bitmap.o
 TEST_BUILTINS_OBJS += test-bloom.o
 TEST_BUILTINS_OBJS += test-bundle-uri.o
 TEST_BUILTINS_OBJS += test-cache-tree.o
+TEST_BUILTINS_OBJS += test-check-connected.o
 TEST_BUILTINS_OBJS += test-chmtime.o
 TEST_BUILTINS_OBJS += test-config.o
 TEST_BUILTINS_OBJS += test-crontab.o
@@ -1357,6 +1358,7 @@ LIB_OBJS += trailer.o
 LIB_OBJS += transport-helper.o
 LIB_OBJS += transport.o
 LIB_OBJS += tree-diff.o
+LIB_OBJS += tree-verify.o
 LIB_OBJS += tree-walk.o
 LIB_OBJS += tree.o
 LIB_OBJS += unpack-trees.o

builtin/rev-list.c

@@ -28,6 +28,7 @@
 #include "commit-reach.h"
 #include "quote.h"
 #include "strbuf.h"
+#include "tree-verify.h"
 
 struct rev_list_info {
 	struct rev_info *revs;
@@ -706,6 +707,7 @@ int cmd_rev_list(int argc,
 	int bisect_find_all = 0;
 	int use_bitmap_index = 0;
 	int filter_provided_objects = 0;
+	int verify_trees_incremental = 0;
 	const char *show_progress = NULL;
 	int ret = 0;
 
@@ -748,6 +750,8 @@ int cmd_rev_list(int argc,
 		if (!strcmp(arg, "--exclude-promisor-objects")) {
 			repo->fetch_if_missing = 0;
 			revs.exclude_promisor_objects = 1;
+		} else if (!strcmp(arg, "--verify-trees-incremental")) {
+			verify_trees_incremental = 1;
 		} else if (skip_prefix(arg, "--missing=", &arg)) {
 			parse_missing_action_value(repo, arg);
 		} else if (!strcmp(arg, "-z")) {
@@ -822,6 +826,8 @@ int cmd_rev_list(int argc,
 
 		if (!strcmp(arg, "--exclude-promisor-objects"))
 			continue; /* already handled above */
+		if (!strcmp(arg, "--verify-trees-incremental"))
+			continue; /* already handled above */
 		if (skip_prefix(arg, "--missing=", &arg))
 			continue; /* already handled above */
 
@@ -935,8 +941,21 @@ int cmd_rev_list(int argc,
 
 	prepare_maximal_independent(&revs);
 
-	if (revs.tree_objects)
+	if (verify_trees_incremental) {
+		struct commit *commit;
+		struct commit_list *new_commits = NULL;
+
+		while ((commit = get_revision(&revs)) != NULL)
+			commit_list_insert(commit, &new_commits);
+
+		ret = verify_commits_incremental(repo, &new_commits,
+						 revs.exclude_promisor_objects);
+		commit_list_free(new_commits);
+		if (ret)
+			goto cleanup;
+	} else if (revs.tree_objects) {
 		mark_edges_uninteresting(&revs, show_edge, 0);
+	}
 
 	if (bisect_list) {
 		int reaches, all;

connected.c

@@ -1,6 +1,7 @@
 #define USE_THE_REPOSITORY_VARIABLE
 
 #include "git-compat-util.h"
+#include "config.h"
 #include "gettext.h"
 #include "hex.h"
 #include "odb.h"
@@ -67,6 +68,26 @@ static int check_connected_promisor(oid_iterate_fn fn,
 	return 1;
 }
 
+static int incremental_check_applicable(struct check_connected_options *opt)
+{
+	const char *algorithm = NULL;
+
+	if (repo_config_get_string_tmp(the_repository,
+				       "transfer.connectivitycheck",
+				       &algorithm))
+		return 0;
+	if (!strcasecmp(algorithm, "full"))
+		return 0;
+	if (strcasecmp(algorithm, "incremental"))
+		die(_("unknown transfer.connectivityCheck algorithm '%s'"),
+		    algorithm);
+
+	if (opt->is_deepening_fetch)
+		return 0;
+
+	return 1;
+}
+
 /*
  * If we feed all the commits we want to verify to this command
  *
@@ -133,6 +154,9 @@ int check_connected(oid_iterate_fn fn, void *cb_data,
 	if (opt->progress)
 		strvec_pushf(&rev_list.args, "--progress=%s",
 			     _("Checking connectivity"));
+	if (incremental_check_applicable(opt))
+		strvec_push(&rev_list.args,
+			    "--verify-trees-incremental");
 
 	rev_list.git_cmd = 1;
 	if (opt->env)

meson.build

@@ -562,6 +562,7 @@ libgit_sources = [
   'transport-helper.c',
   'transport.c',
   'tree-diff.c',
+  'tree-verify.c',
   'tree-walk.c',
   'tree.c',
   'unpack-trees.c',

t/helper/meson.build

@@ -5,6 +5,7 @@ test_tool_sources = [
   'test-bloom.c',
   'test-bundle-uri.c',
   'test-cache-tree.c',
+  'test-check-connected.c',
   'test-chmtime.c',
   'test-config.c',
   'test-crontab.c',

t/helper/test-check-connected.c

@@ -0,0 +1,63 @@
+#define USE_THE_REPOSITORY_VARIABLE
+
+#include "test-tool.h"
+#include "git-compat-util.h"
+#include "hex.h"
+#include "connected.h"
+#include "oid-array.h"
+#include "setup.h"
+
+struct cb_data {
+	struct oid_array *oids;
+	size_t idx;
+};
+
+static const struct object_id *iterate_oids(void *data)
+{
+	struct cb_data *cb = data;
+	if (cb->idx >= cb->oids->nr)
+		return NULL;
+	return &cb->oids->oid[cb->idx++];
+}
+
+int cmd__check_connected(int argc, const char **argv)
+{
+	struct oid_array oids = OID_ARRAY_INIT;
+	struct check_connected_options opt = CHECK_CONNECTED_INIT;
+	struct cb_data cb;
+	int i, ret;
+
+	setup_git_directory(the_repository);
+
+	for (i = 1; i < argc; i++) {
+		struct object_id oid;
+		if (!strcmp(argv[i], "--err-file")) {
+			if (++i >= argc)
+				die("--err-file requires a path argument");
+			opt.err_fd = open(argv[i],
+					  O_WRONLY | O_CREAT | O_TRUNC, 0644);
+			if (opt.err_fd < 0)
+				die_errno("could not open '%s'", argv[i]);
+			continue;
+		}
+		if (!strcmp(argv[i], "--quiet")) {
+			opt.quiet = 1;
+			continue;
+		}
+		if (get_oid_hex(argv[i], &oid))
+			die("not a valid object: %s", argv[i]);
+		oid_array_append(&oids, &oid);
+	}
+
+	if (!oids.nr)
+		die("usage: test-tool check-connected"
+		    " [--err-file <path>]"
+		    " [--quiet] <oid>...");
+
+	cb.oids = &oids;
+	cb.idx = 0;
+
+	ret = check_connected(iterate_oids, &cb, &opt);
+	oid_array_clear(&oids);
+	return !!ret;
+}

t/helper/test-tool.c

@@ -15,6 +15,7 @@ static struct test_cmd cmds[] = {
 	{ "bloom", cmd__bloom },
 	{ "bundle-uri", cmd__bundle_uri },
 	{ "cache-tree", cmd__cache_tree },
+	{ "check-connected", cmd__check_connected },
 	{ "chmtime", cmd__chmtime },
 	{ "config", cmd__config },
 	{ "crontab", cmd__crontab },

t/helper/test-tool.h

@@ -8,6 +8,7 @@ int cmd__bitmap(int argc, const char **argv);
 int cmd__bloom(int argc, const char **argv);
 int cmd__bundle_uri(int argc, const char **argv);
 int cmd__cache_tree(int argc, const char **argv);
+int cmd__check_connected(int argc, const char **argv);
 int cmd__chmtime(int argc, const char **argv);
 int cmd__config(int argc, const char **argv);
 int cmd__crontab(int argc, const char **argv);

t/meson.build

@@ -652,6 +652,7 @@ integration_tests = [
   't5409-colorize-remote-messages.sh',
   't5410-receive-pack.sh',
   't5411-proc-receive-hook.sh',
+  't5412-connectivity-check.sh',
   't5500-fetch-pack.sh',
   't5501-fetch-push-alternates.sh',
   't5502-quickfetch.sh',

t/perf/p5412-connectivity-check.sh

@@ -0,0 +1,122 @@
+#!/bin/sh
+
+test_description='performance of connectivity check modes
+
+Compare the default and incremental rev-list connectivity modes
+using test-tool check-connected directly, avoiding pack transfer
+noise.
+
+Each repository has a flat tree of many directories with 100 files
+in each.  Three axes are scaled independently: tree size, commit
+count, and files changed per commit.'
+
+. ./perf-lib.sh
+
+test_perf_fresh_repo
+
+# Output a fast-import stream for the initial tree: one commit
+# with $1 directories, each containing $2 files.
+create_tree () {
+	perl -e '
+		my ($nd, $nf) = @ARGV;
+		printf "commit refs/heads/main\n";
+		printf "committer perf <perf\@test.com> now\n";
+		printf "data 7\ninitial\n";
+		for my $d (1..$nd) {
+			for my $f (1..$nf) {
+				printf "M 100644 inline d-%04d/f-%03d\n", $d, $f;
+				printf "data 7\n%03d%03d\n", $d, $f;
+			}
+		}
+	' "$@"
+}
+
+# Output a fast-import stream for $1 commits, each modifying one
+# file in a directory chosen by cycling through 1..$3 (out of $2
+# total directories).
+create_commits () {
+	perl -e '
+		my ($nc, $nd, $hot, $files_per_commit) = @ARGV;
+		$hot = $nd if !$hot || $hot > $nd;
+		$files_per_commit = 1 if !$files_per_commit;
+		for my $i (1..$nc) {
+			my $msg = sprintf "change-%03d", $i;
+			printf "commit refs/heads/main\n";
+			printf "committer perf <perf\@test.com> now\n";
+			printf "data %d\n%s\n", length($msg), $msg;
+			printf "from refs/heads/main^0\n" if $i == 1;
+			for my $j (0..$files_per_commit-1) {
+				my $d = (($i + $j) % $hot) + 1;
+				my $content = sprintf "c%d-%d", $i, $j;
+				printf "M 100644 inline d-%04d/f-001\n", $d;
+				printf "data %d\n%s\n", length($content), $content;
+			}
+		}
+	' "$@"
+}
+
+# $1=dirs  $2=files_per_dir  $3=commits  $4=hot_dirs (optional, default=all)
+# $5=files_per_commit (optional, default=1)
+test_perf_conn () {
+	local nd="$1" nf="$2" nc="$3" hot="${4:-$1}" fpc="${5:-1}"
+	local total=$(($nd * $nf))
+	local name="repo-${nd}d-${nf}f-${nc}c-${hot}h-${fpc}fpc"
+	local label="${total} files, ${nc} commits"
+	if test "$hot" -lt "$nd"
+	then
+		label="$label (${hot} hot dirs)"
+	fi
+	if test "$fpc" -gt 1
+	then
+		label="$label (${fpc} files/commit)"
+	fi
+
+	test_expect_success "setup $label" '
+		git init '"$name"' &&
+		create_tree '"$nd"' '"$nf"' |
+		git -C '"$name"' fast-import --date-format=now --quiet &&
+		git -C '"$name"' rev-parse main >'"${name}"'_old &&
+		create_commits '"$nc"' '"$nd"' '"$hot"' '"$fpc"' |
+		git -C '"$name"' fast-import --date-format=now --quiet &&
+		git -C '"$name"' rev-parse main >'"${name}"'_new &&
+		git -C '"$name"' update-ref refs/heads/main \
+			$(cat '"${name}"'_old) &&
+		git -C '"$name"' repack -ad &&
+		git -C '"$name"' config gc.auto 0
+	'
+
+	for mode in full incremental
+	do
+		test_perf "$label ($mode)" \
+			--setup '
+			git -C '"$name"' config transfer.connectivityCheck '"$mode" \
+			'
+			test-tool -C '"$name"' check-connected $(cat '"${name}"'_new)
+		'
+	done
+}
+
+# Scaling tree size (10 commits, 10 files/commit).
+test_perf_conn   50 100 10   50 10
+test_perf_conn  500 100 10  500 10
+test_perf_conn 2000 100 10 2000 10
+test_perf_conn 8000 100 10 8000 10
+
+# Scaling commit count (200K files, 10 files/commit).
+test_perf_conn 2000 100    1 2000 10
+test_perf_conn 2000 100   10 2000 10
+test_perf_conn 2000 100  100 2000 10
+test_perf_conn 2000 100   500 2000 10
+test_perf_conn 2000 100  3000 2000 10
+test_perf_conn 2000 100  5000 2000 10
+test_perf_conn 2000 100 10000 2000 10
+
+# Scaling files per commit (200K files, 10 commits).
+test_perf_conn 2000 100 10 2000   1
+test_perf_conn 2000 100 10 2000  10
+test_perf_conn 2000 100 10 2000  100
+test_perf_conn 2000 100 10 2000  500
+test_perf_conn 2000 100 10 2000 1000
+test_perf_conn 2000 100 10 2000 2000
+
+test_done