diff --git a/Documentation/RelNotes/2.56.0.adoc b/Documentation/RelNotes/2.56.0.adoc index 9e3bf38f5e5a74..811f74bc7dbcb6 100644 --- a/Documentation/RelNotes/2.56.0.adoc +++ b/Documentation/RelNotes/2.56.0.adoc @@ -48,6 +48,14 @@ UI, Workflows & Features help option ('-h' or '--help') is requested directly by the user, aligning with standard Unix convention. + * The '[includeIf "condition"]' conditional inclusion facility for + configuration files has been taught to use the location of the + worktree in its condition. + + * The usage string and SYNOPSIS for 'git fast-export' have been + standardized to make them consistent with each other and with other + commands. + Performance, Internal Implementation, Development Support etc. -------------------------------------------------------------- @@ -167,6 +175,37 @@ Performance, Internal Implementation, Development Support etc. 'change-id' trailer, ensuring that sent tags generated by 'b4' contain the required tracking information for subsequent runs. + * 'git receive-pack' has been refactored to use ODB transaction + interfaces instead of directly managing 'tmp_objdir' for staging + incoming objects, bringing it closer to being ODB backend agnostic. + + * The test script 't/t9811-git-p4-label-import.sh' has been + modernized to use 'test_path_is_file' and 'test_path_is_missing' + instead of raw 'test -f' and '! test -f' calls. + + * A redundant strbuf_reset() call in the 'HAVE_GETDELIM' path of + strbuf_getwholeline() has been removed, as getdelim() overwrites the + buffer and the length is updated afterward. + + * The object database enumeration interface odb_for_each_object() has + been taught to accept object filters, allowing the underlying backends + to optimize the traversal by using reachability bitmaps when + available. 'git cat-file --batch-all-objects' has been updated to use + this generic interface, simplifying its code and avoiding direct + access to ODB backend internals. + + * The test script 't/t1100-commit-tree-options.sh' has been modernized + by converting test cases to the modern style (using single quotes and + tab indentation) and moving the creation of the expected file inside + the setup test so it runs under the protection of the test harness. + + * The test script 't/t7614-merge-signoff.sh' has been updated to avoid + suppressing the exit code of 'git' commands in a pipe. + + * The 'git rev-list --no-walk' command has been corrected to restore + pathspec filtering, which was lost when the streaming walk was + refactored. + Fixes since v2.55 ----------------- @@ -306,3 +345,8 @@ Fixes since v2.55 * The stream-based object signature verification path has been corrected to avoid double-closing the stream on read errors. (merge cfd52a74a0 ps/odb-stream-double-close-fix later to maint). + + * The '-i' shorthand for the '--init' option, which was accepted by the + 'git submodule update' command until it was broken in a modernization + of the option-parsing code, has been restored. + (merge ff1da37f58 dm/submodule-update-i-shorthand later to maint). diff --git a/Documentation/config.adoc b/Documentation/config.adoc index 15b1a4d5934758..1ef72de62f2ba6 100644 --- a/Documentation/config.adoc +++ b/Documentation/config.adoc @@ -146,6 +146,51 @@ refer to linkgit:gitignore[5] for details. For convenience: This is the same as `gitdir` except that matching is done case-insensitively (e.g. on case-insensitive file systems) +`worktree`:: + The data that follows the keyword `worktree` and a colon is used as a + glob pattern. If the working directory of the current worktree matches + the pattern, the include condition is met. ++ +The worktree location is the path where files are checked out (as returned +by `git rev-parse --show-toplevel`). This is different from `gitdir`, which +matches the `.git` directory path. In a linked worktree, the worktree path +is the directory where that worktree's files are located, not the main +repository's `.git` directory. ++ +The pattern uses the same glob syntax as `gitdir` (including `~/`, `./`, +`**/`, and trailing-`/` prefix matching). This condition will never match +in a bare repository (which has no worktree). ++ +Unlike `gitdir`, the `worktree` condition currently matches only the +realpath-resolved worktree location. If the working tree was entered via a +symbolic link, a pattern that uses the symbolic-link spelling may not match; +use the real path instead. ++ +This is useful when you want to apply configuration based on where the +working tree is located on the filesystem. For example, a contributor who +works on the same project both personally and as an employee can use +different `user.name` and `user.email` values depending on which directory +the worktree is checked out under: ++ +---- +[includeIf "worktree:/home/user/work/"] + path = ~/.config/git/work.inc +[includeIf "worktree:/home/user/personal/"] + path = ~/.config/git/personal.inc +---- ++ +While `extensions.worktreeConfig` (see linkgit:git-worktree[1]) also supports +per-worktree configuration, it stores the config inside each repository's +`.git/config.worktree` file and requires running `git config --worktree` +inside each worktree individually. In contrast, `includeIf "worktree:..."` +can be set once in a global or system-level configuration file (e.g. +`~/.config/git/config`) and applies to all repositories at once based on +their worktree location. + +`worktree/i`:: + This is the same as `worktree` except that matching is done + case-insensitively (e.g. on case-insensitive file systems) + `onbranch`:: The data that follows the keyword `onbranch` and a colon is taken to be a pattern with standard globbing wildcards and two additional @@ -244,6 +289,14 @@ Example [includeIf "gitdir:~/to/group/"] path = /path/to/foo.inc +; include if the worktree is at /path/to/project-build +[includeIf "worktree:/path/to/project-build"] + path = build-config.inc + +; include for all worktrees inside /path/to/group +[includeIf "worktree:/path/to/group/"] + path = group-config.inc + ; relative paths are always relative to the including ; file (if the condition is true); their location is not ; affected by the condition diff --git a/Documentation/git-fast-export.adoc b/Documentation/git-fast-export.adoc index 297b57bb2efdc2..719aeca244d534 100644 --- a/Documentation/git-fast-export.adoc +++ b/Documentation/git-fast-export.adoc @@ -9,7 +9,7 @@ git-fast-export - Git data exporter SYNOPSIS -------- [verse] -'git fast-export' [] | 'git fast-import' +'git fast-export' [] [] [[--] ...] DESCRIPTION ----------- diff --git a/builtin/add.c b/builtin/add.c index c859f665199efa..60ffbede2be58a 100644 --- a/builtin/add.c +++ b/builtin/add.c @@ -581,7 +581,7 @@ int cmd_add(int argc, string_list_clear(&only_match_skip_worktree, 0); } - transaction = odb_transaction_begin(repo->objects); + odb_transaction_begin_or_die(repo->objects, &transaction, 0); ps_matched = xcalloc(pathspec.nr, 1); if (add_renormalize) diff --git a/builtin/cat-file.c b/builtin/cat-file.c index b4b99a73da9696..1458dd76d683e0 100644 --- a/builtin/cat-file.c +++ b/builtin/cat-file.c @@ -20,7 +20,6 @@ #include "userdiff.h" #include "oid-array.h" #include "packfile.h" -#include "pack-bitmap.h" #include "object-file.h" #include "object-name.h" #include "odb.h" @@ -844,28 +843,6 @@ static int batch_one_object_oi(const struct object_id *oid, return payload->callback(oid, NULL, 0, payload->payload); } -static int batch_one_object_packed(const struct object_id *oid, - struct packed_git *pack, - uint32_t pos, - void *_payload) -{ - struct for_each_object_payload *payload = _payload; - return payload->callback(oid, pack, nth_packed_object_offset(pack, pos), - payload->payload); -} - -static int batch_one_object_bitmapped(const struct object_id *oid, - enum object_type type UNUSED, - int flags UNUSED, - uint32_t hash UNUSED, - struct packed_git *pack, - off_t offset, - void *_payload) -{ - struct for_each_object_payload *payload = _payload; - return payload->callback(oid, pack, offset, payload->payload); -} - static void batch_each_object(struct batch_options *opt, for_each_object_fn callback, unsigned flags, @@ -875,56 +852,17 @@ static void batch_each_object(struct batch_options *opt, .callback = callback, .payload = _payload, }; + struct odb_source_info source_info; + struct object_info oi = { + .source_infop = &source_info, + }; struct odb_for_each_object_options opts = { .flags = flags, + .filter = &opt->objects_filter, }; - struct bitmap_index *bitmap = NULL; - struct odb_source *source; - - /* - * TODO: we still need to tap into implementation details of the object - * database sources. Ideally, we should extend `odb_for_each_object()` - * to handle object filters itself so that we can move the filtering - * logic into the individual sources. - */ - odb_prepare_alternates(the_repository->objects); - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - int ret = odb_source_for_each_object(&files->loose->base, NULL, batch_one_object_oi, - &payload, &opts); - if (ret) - break; - } - - if (opt->objects_filter.choice != LOFC_DISABLED && - (bitmap = prepare_bitmap_git(the_repository)) && - !for_each_bitmapped_object(bitmap, &opt->objects_filter, - batch_one_object_bitmapped, &payload)) { - struct packed_git *pack; - - repo_for_each_pack(the_repository, pack) { - if (bitmap_index_contains_pack(bitmap, pack) || - open_pack_index(pack)) - continue; - for_each_object_in_pack(pack, batch_one_object_packed, - &payload, flags); - } - } else { - struct odb_source_info source_info; - struct object_info oi = { - .source_infop = &source_info, - }; - - for (source = the_repository->objects->sources; source; source = source->next) { - struct odb_source_files *files = odb_source_files_downcast(source); - int ret = odb_source_for_each_object(&files->packed->base, &oi, - batch_one_object_oi, &payload, &opts); - if (ret) - break; - } - } - free_bitmap_index(bitmap); + odb_for_each_object_ext(the_repository->objects, &oi, + batch_one_object_oi, &payload, &opts); } static int batch_objects(struct batch_options *opt) diff --git a/builtin/fast-export.c b/builtin/fast-export.c index 0be43104dc0d23..629d7c591a9d70 100644 --- a/builtin/fast-export.c +++ b/builtin/fast-export.c @@ -33,7 +33,7 @@ #include "gpg-interface.h" static const char *const fast_export_usage[] = { - N_("git fast-export []"), + N_("git fast-export [] [] [[--] ...]"), NULL }; diff --git a/builtin/pack-objects.c b/builtin/pack-objects.c index ea5eab4cf841bf..3673b14b89b275 100644 --- a/builtin/pack-objects.c +++ b/builtin/pack-objects.c @@ -1867,8 +1867,8 @@ static const char no_closure_warning[] = N_( "disabling bitmap writing, as some objects are not being packed" ); -static int add_object_entry(const struct object_id *oid, enum object_type type, - const char *name, int exclude) +static void add_object_entry(const struct object_id *oid, enum object_type type, + const char *name, int exclude) { struct packed_git *found_pack = NULL; off_t found_offset = 0; @@ -1876,7 +1876,7 @@ static int add_object_entry(const struct object_id *oid, enum object_type type, display_progress(progress_state, ++nr_seen); if (have_duplicate_entry(oid, exclude)) - return 0; + return; if (!want_object_in_pack(oid, exclude, &found_pack, &found_offset)) { /* The pack is missing an object, so it will not have closure */ @@ -1885,13 +1885,12 @@ static int add_object_entry(const struct object_id *oid, enum object_type type, warning(_(no_closure_warning)); write_bitmap_index = 0; } - return 0; + return; } create_object_entry(oid, type, pack_name_hash_fn(name), exclude, name && no_try_delta(name), found_pack, found_offset); - return 1; } static int add_object_entry_from_bitmap(const struct object_id *oid, @@ -1909,7 +1908,7 @@ static int add_object_entry_from_bitmap(const struct object_id *oid, return 0; create_object_entry(oid, type, name_hash, 0, 0, pack, offset); - return 1; + return 0; } struct pbase_tree_cache { diff --git a/builtin/receive-pack.c b/builtin/receive-pack.c index faf0f120ac186b..7190d7f43063c9 100644 --- a/builtin/receive-pack.c +++ b/builtin/receive-pack.c @@ -37,7 +37,6 @@ #include "sigchain.h" #include "string-list.h" #include "strvec.h" -#include "tmp-objdir.h" #include "trace.h" #include "trace2.h" #include "version.h" @@ -112,8 +111,6 @@ static enum { } use_keepalive; static int keepalive_in_sec = 5; -static struct tmp_objdir *tmp_objdir; - static struct proc_receive_ref { unsigned int want_add:1, want_delete:1, @@ -926,6 +923,7 @@ static void receive_hook_feed_state_free(void *data) static int run_receive_hook(struct command *commands, const char *hook_name, int skip_broken, + struct odb_transaction *transaction, const struct string_list *push_options) { struct run_hooks_opt opt = RUN_HOOKS_OPT_INIT; @@ -959,8 +957,8 @@ static int run_receive_hook(struct command *commands, strvec_push(&opt.env, "GIT_PUSH_OPTION_COUNT"); } - if (tmp_objdir) - strvec_pushv(&opt.env, tmp_objdir_env(tmp_objdir)); + if (transaction) + odb_transaction_env(transaction, &opt.env); prepare_push_cert_sha1(&opt); @@ -1363,7 +1361,6 @@ static int update_shallow_ref(struct command *cmd, struct shallow_info *si) !delayed_reachability_test(si, i)) oid_array_append(&extra, &si->shallow->oid[i]); - opt.env = tmp_objdir_env(tmp_objdir); setup_alternate_shallow(&shallow_lock, &opt.shallow_file, &extra); if (check_connected(command_singleton_iterator, cmd, &opt)) { rollback_shallow_file(the_repository, &shallow_lock); @@ -1790,24 +1787,30 @@ static const struct object_id *command_singleton_iterator(void *cb_data) } static void set_connectivity_errors(struct command *commands, - struct shallow_info *si) + struct shallow_info *si, + struct odb_transaction *transaction) { struct command *cmd; for (cmd = commands; cmd; cmd = cmd->next) { struct command *singleton = cmd; struct check_connected_options opt = CHECK_CONNECTED_INIT; + struct strvec env = STRVEC_INIT; if (shallow_update && si->shallow_ref[cmd->index]) /* to be checked in update_shallow_ref() */ continue; - opt.env = tmp_objdir_env(tmp_objdir); + odb_transaction_env(transaction, &env); + opt.env = env.v; + if (!check_connected(command_singleton_iterator, &singleton, &opt)) continue; cmd->error_string = "missing necessary objects"; + + strvec_clear(&env); } } @@ -2028,6 +2031,7 @@ static void execute_commands_atomic(struct command *commands, static void execute_commands(struct command *commands, const char *unpacker_error, struct shallow_info *si, + struct odb_transaction *transaction, const struct string_list *push_options) { struct check_connected_options opt = CHECK_CONNECTED_INIT; @@ -2044,6 +2048,8 @@ static void execute_commands(struct command *commands, } if (!skip_connectivity_check) { + struct strvec env = STRVEC_INIT; + if (use_sideband) { memset(&muxer, 0, sizeof(muxer)); muxer.proc = copy_to_sideband; @@ -2057,14 +2063,17 @@ static void execute_commands(struct command *commands, data.si = si; opt.err_fd = err_fd; opt.progress = err_fd && !quiet; - opt.env = tmp_objdir_env(tmp_objdir); + odb_transaction_env(transaction, &env); + opt.env = env.v; opt.exclude_hidden_refs_section = "receive"; if (check_connected(iterate_receive_command_list, &data, &opt)) - set_connectivity_errors(commands, si); + set_connectivity_errors(commands, si, transaction); if (use_sideband) finish_async(&muxer); + + strvec_clear(&env); } reject_updates_to_hidden(commands); @@ -2085,7 +2094,7 @@ static void execute_commands(struct command *commands, } } - if (run_receive_hook(commands, "pre-receive", 0, push_options)) { + if (run_receive_hook(commands, "pre-receive", 0, transaction, push_options)) { for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) cmd->error_string = "pre-receive hook declined"; @@ -2106,14 +2115,13 @@ static void execute_commands(struct command *commands, * Now we'll start writing out refs, which means the objects need * to be in their final positions so that other processes can see them. */ - if (tmp_objdir_migrate(tmp_objdir) < 0) { + if (odb_transaction_commit(transaction)) { for (cmd = commands; cmd; cmd = cmd->next) { if (!cmd->error_string) cmd->error_string = "unable to migrate objects to permanent storage"; } return; } - tmp_objdir = NULL; check_aliased_updates(commands); @@ -2326,7 +2334,8 @@ static void push_header_arg(struct strvec *args, struct pack_header *hdr) ntohl(hdr->hdr_version), ntohl(hdr->hdr_entries)); } -static const char *unpack(int err_fd, struct shallow_info *si) +static const char *unpack(int err_fd, struct shallow_info *si, + struct odb_transaction *transaction) { struct pack_header hdr; const char *hdr_err; @@ -2351,20 +2360,7 @@ static const char *unpack(int err_fd, struct shallow_info *si) strvec_push(&child.args, alt_shallow_file); } - tmp_objdir = tmp_objdir_create(the_repository, "incoming"); - if (!tmp_objdir) { - if (err_fd > 0) - close(err_fd); - return "unable to create temporary object directory"; - } - strvec_pushv(&child.env, tmp_objdir_env(tmp_objdir)); - - /* - * Normally we just pass the tmp_objdir environment to the child - * processes that do the heavy lifting, but we may need to see these - * objects ourselves to set up shallow information. - */ - tmp_objdir_add_as_alternate(tmp_objdir); + odb_transaction_env(transaction, &child.env); if (ntohl(hdr.hdr_entries) < unpack_limit) { strvec_push(&child.args, "unpack-objects"); @@ -2431,13 +2427,14 @@ static const char *unpack(int err_fd, struct shallow_info *si) return NULL; } -static const char *unpack_with_sideband(struct shallow_info *si) +static const char *unpack_with_sideband(struct shallow_info *si, + struct odb_transaction *transaction) { struct async muxer; const char *ret; if (!use_sideband) - return unpack(0, si); + return unpack(0, si, transaction); use_keepalive = KEEPALIVE_AFTER_NUL; memset(&muxer, 0, sizeof(muxer)); @@ -2446,7 +2443,7 @@ static const char *unpack_with_sideband(struct shallow_info *si) if (start_async(&muxer)) return NULL; - ret = unpack(muxer.in, si); + ret = unpack(muxer.in, si, transaction); finish_async(&muxer); return ret; @@ -2623,6 +2620,7 @@ int cmd_receive_pack(int argc, struct oid_array ref = OID_ARRAY_INIT; struct shallow_info si; struct packet_reader reader; + struct odb_transaction *transaction = NULL; struct option options[] = { OPT__QUIET(&quiet, N_("quiet")), @@ -2707,11 +2705,14 @@ int cmd_receive_pack(int argc, if (!si.nr_ours && !si.nr_theirs) shallow_update = 0; if (!delete_only(commands)) { - unpack_status = unpack_with_sideband(&si); + if (odb_transaction_begin(the_repository->objects, &transaction, ODB_TRANSACTION_RECEIVE)) + unpack_status = "unable to start object transaction"; + else + unpack_status = unpack_with_sideband(&si, transaction); update_shallow_info(commands, &si, &ref); } use_keepalive = KEEPALIVE_ALWAYS; - execute_commands(commands, unpack_status, &si, + execute_commands(commands, unpack_status, &si, transaction, &push_options); delete_tempfile(&pack_lockfile); sigchain_push(SIGPIPE, SIG_IGN); @@ -2720,7 +2721,7 @@ int cmd_receive_pack(int argc, else if (report_status) report(commands, unpack_status); sigchain_pop(SIGPIPE); - run_receive_hook(commands, "post-receive", 1, + run_receive_hook(commands, "post-receive", 1, NULL, &push_options); run_update_post_hook(commands); free_commands(commands); diff --git a/builtin/rev-list.c b/builtin/rev-list.c index 8f63003709242e..02818b81c63fd7 100644 --- a/builtin/rev-list.c +++ b/builtin/rev-list.c @@ -486,7 +486,7 @@ static int show_object_fast( void *payload UNUSED) { fprintf(stdout, "%s\n", oid_to_hex(oid)); - return 1; + return 0; } static void print_disk_usage(off_t size) diff --git a/builtin/submodule--helper.c b/builtin/submodule--helper.c index 510f193a15b682..2b3c9762afec0a 100644 --- a/builtin/submodule--helper.c +++ b/builtin/submodule--helper.c @@ -2990,7 +2990,7 @@ static int module_update(int argc, const char **argv, const char *prefix, struct option module_update_options[] = { OPT__SUPER_PREFIX(&opt.super_prefix), OPT__FORCE(&opt.force, N_("force checkout updates"), 0), - OPT_BOOL(0, "init", &opt.init, + OPT_BOOL('i', "init", &opt.init, N_("initialize uninitialized submodules before update")), OPT_BOOL(0, "remote", &opt.remote, N_("use SHA-1 of submodule's remote tracking branch")), diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index 93a9caa58269ab..4263edfbecdd39 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -598,7 +598,7 @@ static void unpack_all(void) progress = start_progress(the_repository, _("Unpacking objects"), nr_objects); CALLOC_ARRAY(obj_list, nr_objects); - transaction = odb_transaction_begin(the_repository->objects); + odb_transaction_begin_or_die(the_repository->objects, &transaction, 0); for (i = 0; i < nr_objects; i++) { unpack_one(i); display_progress(progress, i + 1); diff --git a/builtin/update-index.c b/builtin/update-index.c index 5d80d4827d8aa1..4c4b39a1575a38 100644 --- a/builtin/update-index.c +++ b/builtin/update-index.c @@ -1124,7 +1124,7 @@ int cmd_update_index(int argc, * Allow the object layer to optimize adding multiple objects in * a batch. */ - transaction = odb_transaction_begin(the_repository->objects); + odb_transaction_begin_or_die(the_repository->objects, &transaction, 0); while (ctx.argc) { if (parseopt_state != PARSE_OPT_DONE) parseopt_state = parse_options_step(&ctx, options, diff --git a/cache-tree.c b/cache-tree.c index 9204eb3ac7a120..d92f5132865f13 100644 --- a/cache-tree.c +++ b/cache-tree.c @@ -516,6 +516,7 @@ static int update_one(struct cache_tree *it, int cache_tree_update(struct index_state *istate, int flags) { + int inflight = !!the_repository->objects->transaction; struct odb_transaction *transaction; int skip, i; @@ -532,10 +533,12 @@ int cache_tree_update(struct index_state *istate, int flags) trace_performance_enter(); trace2_region_enter("cache_tree", "update", istate->repo); - transaction = odb_transaction_begin(the_repository->objects); + if (!inflight) + odb_transaction_begin_or_die(the_repository->objects, &transaction, 0); i = update_one(istate->cache_tree, istate->cache, istate->cache_nr, "", 0, &skip, flags); - odb_transaction_commit(transaction); + if (!inflight) + odb_transaction_commit(transaction); trace2_region_leave("cache_tree", "update", istate->repo); trace_performance_leave("cache_tree_update"); if (i < 0) diff --git a/config.c b/config.c index 6a0de86e3ae958..9d6d7872d76c10 100644 --- a/config.c +++ b/config.c @@ -235,23 +235,20 @@ static int prepare_include_condition_pattern(const struct key_value_info *kvi, return 0; } -static int include_by_gitdir(const struct key_value_info *kvi, - const struct config_options *opts, - const char *cond, size_t cond_len, int icase) +static int include_by_path(const struct key_value_info *kvi, + const char *path, + const char *cond, size_t cond_len, int icase) { struct strbuf text = STRBUF_INIT; struct strbuf pattern = STRBUF_INIT; size_t prefix; int ret = 0; - const char *git_dir; int already_tried_absolute = 0; - if (opts->git_dir) - git_dir = opts->git_dir; - else + if (!path) goto done; - strbuf_realpath(&text, git_dir, 1); + strbuf_realpath(&text, path, 1); strbuf_add(&pattern, cond, cond_len); ret = prepare_include_condition_pattern(kvi, &pattern, &prefix); if (ret < 0) @@ -284,7 +281,7 @@ static int include_by_gitdir(const struct key_value_info *kvi, * which'll do the right thing */ strbuf_reset(&text); - strbuf_add_absolute_path(&text, git_dir); + strbuf_add_absolute_path(&text, path); already_tried_absolute = 1; goto again; } @@ -400,9 +397,15 @@ static int include_condition_is_true(const struct key_value_info *kvi, const struct config_options *opts = inc->opts; if (skip_prefix_mem(cond, cond_len, "gitdir:", &cond, &cond_len)) - return include_by_gitdir(kvi, opts, cond, cond_len, 0); + return include_by_path(kvi, opts->git_dir, cond, cond_len, 0); else if (skip_prefix_mem(cond, cond_len, "gitdir/i:", &cond, &cond_len)) - return include_by_gitdir(kvi, opts, cond, cond_len, 1); + return include_by_path(kvi, opts->git_dir, cond, cond_len, 1); + else if (skip_prefix_mem(cond, cond_len, "worktree:", &cond, &cond_len)) + return include_by_path(kvi, inc->repo ? repo_get_work_tree(inc->repo) : NULL, + cond, cond_len, 0); + else if (skip_prefix_mem(cond, cond_len, "worktree/i:", &cond, &cond_len)) + return include_by_path(kvi, inc->repo ? repo_get_work_tree(inc->repo) : NULL, + cond, cond_len, 1); else if (skip_prefix_mem(cond, cond_len, "onbranch:", &cond, &cond_len)) return include_by_branch(inc, cond, cond_len); else if (skip_prefix_mem(cond, cond_len, "hasconfig:remote.*.url:", &cond, diff --git a/object-file.c b/object-file.c index 2d84f0871363e6..7ff2b730ac0ead 100644 --- a/object-file.c +++ b/object-file.c @@ -27,6 +27,7 @@ #include "path.h" #include "read-cache-ll.h" #include "setup.h" +#include "strvec.h" #include "tempfile.h" #include "tmp-objdir.h" @@ -494,9 +495,10 @@ struct odb_transaction_files { struct tmp_objdir *objdir; struct transaction_packfile packfile; + const char *prefix; }; -static void prepare_loose_object_transaction(struct odb_transaction *base) +static int odb_transaction_files_prepare(struct odb_transaction *base) { struct odb_transaction_files *transaction = container_of_or_null(base, struct odb_transaction_files, base); @@ -508,19 +510,28 @@ static void prepare_loose_object_transaction(struct odb_transaction *base) * added at the time they call odb_transaction_files_begin. */ if (!transaction || transaction->objdir) - return; + return 0; + + transaction->objdir = tmp_objdir_create(base->source->odb->repo, transaction->prefix); + if (!transaction->objdir) + return error(_("unable to create temporary object directory")); - transaction->objdir = tmp_objdir_create(base->source->odb->repo, "bulk-fsync"); - if (transaction->objdir) - tmp_objdir_replace_primary_odb(transaction->objdir, 0); + tmp_objdir_replace_primary_odb(transaction->objdir, 0); + + return 0; } -static void fsync_loose_object_transaction(struct odb_transaction *base, - int fd, const char *filename) +static void odb_transaction_files_fsync(struct odb_transaction *base, + int fd, const char *filename) { struct odb_transaction_files *transaction = container_of_or_null(base, struct odb_transaction_files, base); + if (!transaction || !transaction->objdir) { + fsync_or_die(fd, filename); + return; + } + /* * If we have an active ODB transaction, we issue a call that * cleans the filesystem page cache but avoids a hardware flush @@ -528,49 +539,13 @@ static void fsync_loose_object_transaction(struct odb_transaction *base, * before renaming the objects to their final names as part of * flush_batch_fsync. */ - if (!transaction || !transaction->objdir || - git_fsync(fd, FSYNC_WRITEOUT_ONLY) < 0) { + if (git_fsync(fd, FSYNC_WRITEOUT_ONLY) < 0) { if (errno == ENOSYS) warning(_("core.fsyncMethod = batch is unsupported on this platform")); fsync_or_die(fd, filename); } } -/* - * Cleanup after batch-mode fsync_object_files. - */ -static void flush_loose_object_transaction(struct odb_transaction_files *transaction) -{ - struct strbuf temp_path = STRBUF_INIT; - struct tempfile *temp; - - if (!transaction->objdir) - return; - - /* - * Issue a full hardware flush against a temporary file to ensure - * that all objects are durable before any renames occur. The code in - * fsync_loose_object_transaction has already issued a writeout - * request, but it has not flushed any writeback cache in the storage - * hardware or any filesystem logs. This fsync call acts as a barrier - * to ensure that the data in each new object file is durable before - * the final name is visible. - */ - strbuf_addf(&temp_path, "%s/bulk_fsync_XXXXXX", - repo_get_object_directory(transaction->base.source->odb->repo)); - temp = xmks_tempfile(temp_path.buf); - fsync_or_die(get_tempfile_fd(temp), get_tempfile_path(temp)); - delete_tempfile(&temp); - strbuf_release(&temp_path); - - /* - * Make the object files visible in the primary ODB after their data is - * fully durable. - */ - tmp_objdir_migrate(transaction->objdir); - transaction->objdir = NULL; -} - /* Finalize a file on disk, and close it. */ static void close_loose_object(struct odb_source_loose *loose, int fd, const char *filename) @@ -579,7 +554,7 @@ static void close_loose_object(struct odb_source_loose *loose, goto out; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) - fsync_loose_object_transaction(loose->base.odb->transaction, fd, filename); + odb_transaction_files_fsync(loose->base.odb->transaction, fd, filename); else if (fsync_object_files > 0) fsync_or_die(fd, filename); else @@ -758,7 +733,7 @@ int write_loose_object(struct odb_source_loose *loose, static struct strbuf filename = STRBUF_INIT; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) - prepare_loose_object_transaction(loose->base.odb->transaction); + odb_transaction_files_prepare(loose->base.odb->transaction); odb_loose_path(loose, &filename, oid); @@ -822,7 +797,7 @@ int odb_source_loose_write_stream(struct odb_source_loose *loose, int hdrlen; if (batch_fsync_enabled(FSYNC_COMPONENT_LOOSE_OBJECT)) - prepare_loose_object_transaction(loose->base.odb->transaction); + odb_transaction_files_prepare(loose->base.odb->transaction); /* Since oid is not determined, save tmp file to odb path. */ strbuf_addf(&filename, "%s/", loose->base.path); @@ -1380,13 +1355,17 @@ int index_fd(struct index_state *istate, struct object_id *oid, if (flags & INDEX_WRITE_OBJECT) { struct object_database *odb = the_repository->objects; - struct odb_transaction *transaction = odb_transaction_begin(odb); + struct odb_transaction *transaction = odb->transaction; + int inflight = !!transaction; - ret = odb_transaction_write_object_stream(odb->transaction, + if (!inflight) + odb_transaction_begin_or_die(odb, &transaction, 0); + ret = odb_transaction_write_object_stream(transaction, &stream, xsize_t(st->st_size), oid); - odb_transaction_commit(transaction); + if (!inflight) + odb_transaction_commit(transaction); } else { ret = hash_blob_stream(&stream, the_repository->hash_algo, oid, @@ -1671,27 +1650,93 @@ int read_loose_object(struct repository *repo, return ret; } -static void odb_transaction_files_commit(struct odb_transaction *base) +static int odb_transaction_files_commit(struct odb_transaction *base) { struct odb_transaction_files *transaction = container_of(base, struct odb_transaction_files, base); - flush_loose_object_transaction(transaction); + if (transaction->objdir) { + struct strbuf temp_path = STRBUF_INIT; + struct tempfile *temp; + + /* + * Issue a full hardware flush against a temporary file to ensure + * that all objects are durable before any renames occur. The code in + * odb_transaction_files_fsync has already issued a writeout + * request, but it has not flushed any writeback cache in the storage + * hardware or any filesystem logs. This fsync call acts as a barrier + * to ensure that the data in each new object file is durable before + * the final name is visible. + */ + strbuf_addf(&temp_path, "%s/bulk_fsync_XXXXXX", + repo_get_object_directory(transaction->base.source->odb->repo)); + temp = xmks_tempfile(temp_path.buf); + fsync_or_die(get_tempfile_fd(temp), get_tempfile_path(temp)); + delete_tempfile(&temp); + strbuf_release(&temp_path); + + /* + * Make the object files visible in the primary ODB after their data is + * fully durable. + */ + if (tmp_objdir_migrate(transaction->objdir)) + return error(_("unable to migrate temporary objects")); + + transaction->objdir = NULL; + } + flush_packfile_transaction(transaction); + + return 0; } -struct odb_transaction *odb_transaction_files_begin(struct odb_source *source) +static int odb_transaction_files_env(struct odb_transaction *base, + struct strvec *env) { - struct odb_transaction_files *transaction; - struct object_database *odb = source->odb; + struct odb_transaction_files *transaction = + container_of(base, struct odb_transaction_files, base); + int ret; - if (odb->transaction) - return NULL; + ret = odb_transaction_files_prepare(&transaction->base); + if (!ret) + strvec_pushv(env, tmp_objdir_env(transaction->objdir)); + + return ret; +} + +int odb_transaction_files_begin(struct odb_source *source, + struct odb_transaction **out, + enum odb_transaction_flags flags) +{ + struct odb_transaction_files *transaction; transaction = xcalloc(1, sizeof(*transaction)); transaction->base.source = source; transaction->base.commit = odb_transaction_files_commit; transaction->base.write_object_stream = odb_transaction_files_write_object_stream; + transaction->base.env = odb_transaction_files_env; - return &transaction->base; + transaction->prefix = "bulk-fsync"; + if (flags & ODB_TRANSACTION_RECEIVE) { + /* + * ODB transactions for git-receive-pack(1) eagerly create a + * temporary directory and use a different temporary directory + * prefix. + * + * NEEDSWORK: This transaction flag is only used by the "files" + * backend to special case temporary directory set up and + * handling. Ideally transaction users should not have to care + * though. To avoid this, we could eagerly create the temporary + * directory and use the same prefix name for all transactions. + */ + transaction->prefix = "incoming"; + if (odb_transaction_files_prepare(&transaction->base)) { + free(transaction); + return -1; + } + } + + *out = &transaction->base; + + return 0; } diff --git a/object-file.h b/object-file.h index 4c87cd160bb58a..d04ffa6493ff17 100644 --- a/object-file.h +++ b/object-file.h @@ -5,6 +5,7 @@ #include "object.h" #include "odb.h" #include "odb/source-loose.h" +#include "odb/transaction.h" /* The maximum size for an object header. */ #define MAX_HEADER_LEN 32 @@ -194,9 +195,10 @@ struct odb_transaction; /* * Tell the object database to optimize for adding * multiple objects. odb_transaction_files_commit must be called - * to make new objects visible. If a transaction is already - * pending, NULL is returned. + * to make new objects visible. */ -struct odb_transaction *odb_transaction_files_begin(struct odb_source *source); +int odb_transaction_files_begin(struct odb_source *source, + struct odb_transaction **out, + enum odb_transaction_flags flags); #endif /* OBJECT_FILE_H */ diff --git a/odb.h b/odb.h index 94754643d24997..7995bed97bc36e 100644 --- a/odb.h +++ b/odb.h @@ -8,6 +8,7 @@ #include "thread-utils.h" struct cached_object_entry; +struct list_objects_filter_options; struct odb_source_inmemory; struct packed_git; struct repository; @@ -502,6 +503,17 @@ struct odb_for_each_object_options { */ const struct object_id *prefix; size_t prefix_hex_len; + + /* + * Optional object filter that allows backends to skip yielding + * objects that are excluded by the filter as an optimization. The + * filter is a best-effort hint: backends may use it to skip + * excluded objects (e.g. by consulting a reachability bitmap), but + * are also free to ignore it entirely and yield every object. As a + * consequence, callers must re-apply the filter on yielded objects + * if they require strict filtering semantics. + */ + const struct list_objects_filter_options *filter; }; /* diff --git a/odb/source-files.c b/odb/source-files.c index 6c8e935c753983..413875851135a5 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -181,13 +181,10 @@ static int odb_source_files_write_object_stream(struct odb_source *source, } static int odb_source_files_begin_transaction(struct odb_source *source, - struct odb_transaction **out) + struct odb_transaction **out, + enum odb_transaction_flags flags) { - struct odb_transaction *tx = odb_transaction_files_begin(source); - if (!tx) - return -1; - *out = tx; - return 0; + return odb_transaction_files_begin(source, out, flags); } static int odb_source_files_read_alternates(struct odb_source *source, diff --git a/odb/source-inmemory.c b/odb/source-inmemory.c index f41a602e641897..e47bfd8fccabf0 100644 --- a/odb/source-inmemory.c +++ b/odb/source-inmemory.c @@ -305,7 +305,8 @@ static int odb_source_inmemory_freshen_object(struct odb_source *source, } static int odb_source_inmemory_begin_transaction(struct odb_source *source UNUSED, - struct odb_transaction **out UNUSED) + struct odb_transaction **out UNUSED, + enum odb_transaction_flags flags UNUSED) { return error("in-memory source does not support transactions"); } diff --git a/odb/source-loose.c b/odb/source-loose.c index 091e8e00a77bf4..3f7d04a56e36ce 100644 --- a/odb/source-loose.c +++ b/odb/source-loose.c @@ -638,7 +638,8 @@ static int odb_source_loose_write_object_stream(struct odb_source *source, } static int odb_source_loose_begin_transaction(struct odb_source *source UNUSED, - struct odb_transaction **out UNUSED) + struct odb_transaction **out UNUSED, + enum odb_transaction_flags flags UNUSED) { /* TODO: this is a known omission that we'll want to address eventually. */ return error("loose source does not support transactions"); diff --git a/odb/source-packed.c b/odb/source-packed.c index 647f23cdb9edc9..d00aec5a452c58 100644 --- a/odb/source-packed.c +++ b/odb/source-packed.c @@ -3,11 +3,13 @@ #include "chdir-notify.h" #include "dir.h" #include "git-zlib.h" +#include "list-objects-filter-options.h" #include "mergesort.h" #include "midx.h" #include "odb/source-packed.h" #include "odb/streaming.h" #include "packfile.h" +#include "pack-bitmap.h" static int find_pack_entry(struct odb_source_packed *store, const struct object_id *oid, @@ -143,7 +145,7 @@ static bool should_exclude_pack(struct packed_git *p, enum odb_for_each_object_f } static int for_each_prefixed_object_in_midx( - struct odb_source_packed *store, + struct odb_source_packed *source, struct multi_pack_index *m, const struct odb_for_each_object_options *opts, struct odb_source_packed_for_each_object_wrapper_data *data) @@ -170,6 +172,7 @@ static int for_each_prefixed_object_in_midx( */ for (i = first; i < num; i++) { const struct object_id *current = NULL; + struct packed_git *pack; struct object_id oid; current = nth_midxed_object_oid(&oid, m, i); @@ -177,9 +180,8 @@ static int for_each_prefixed_object_in_midx( if (!match_hash(len, opts->prefix->hash, current->hash)) break; - if (opts->flags) { + if (opts->flags || data->request) { uint32_t pack_id = nth_midxed_pack_int_id(m, i); - struct packed_git *pack; if (prepare_midx_pack(m, pack_id)) { pack_errors = true; @@ -193,9 +195,9 @@ static int for_each_prefixed_object_in_midx( if (data->request) { struct object_info oi = *data->request; + off_t offset = nth_midxed_offset(m, i); - ret = odb_source_read_object_info(&store->base, current, - &oi, 0); + ret = packed_object_info(source, pack, offset, &oi); if (ret) goto out; @@ -219,7 +221,7 @@ static int for_each_prefixed_object_in_midx( } static int for_each_prefixed_object_in_pack( - struct odb_source_packed *store, + struct odb_source_packed *source, struct packed_git *p, const struct odb_for_each_object_options *opts, struct odb_source_packed_for_each_object_wrapper_data *data) @@ -246,8 +248,9 @@ static int for_each_prefixed_object_in_pack( if (data->request) { struct object_info oi = *data->request; + off_t offset = nth_packed_object_offset(p, i); - ret = odb_source_read_object_info(&store->base, &oid, &oi, 0); + ret = packed_object_info(source, p, offset, &oi); if (ret) goto out; @@ -314,6 +317,37 @@ static int odb_source_packed_for_each_prefixed_object( return ret; } +struct bitmapped_for_each_object_data { + struct odb_source_packed *packed; + const struct object_info *request; + const struct odb_for_each_object_options *opts; + odb_for_each_object_cb cb; + void *cb_data; +}; + +static int bitmapped_for_each_object(const struct object_id *oid, + enum object_type type UNUSED, + int flags UNUSED, + uint32_t hash UNUSED, + struct packed_git *pack, + off_t offset, + void *cb_data) +{ + struct bitmapped_for_each_object_data *data = cb_data; + + if (should_exclude_pack(pack, data->opts->flags)) + return 0; + + if (data->request) { + struct object_info oi = *data->request; + if (packed_object_info(data->packed, pack, offset, &oi) < 0) + return -1; + return data->cb(oid, &oi, data->cb_data); + } + + return data->cb(oid, NULL, data->cb_data); +} + static int odb_source_packed_for_each_object(struct odb_source *source, const struct object_info *request, odb_for_each_object_cb cb, @@ -327,12 +361,33 @@ static int odb_source_packed_for_each_object(struct odb_source *source, .cb = cb, .cb_data = cb_data, }; + struct bitmap_index *bitmap = NULL; struct packfile_list_entry *e; int pack_errors = 0, ret; if (opts->prefix) return odb_source_packed_for_each_prefixed_object(packed, opts, &data); + if (opts->filter && + opts->filter->choice != LOFC_DISABLED && + can_filter_bitmap(opts->filter)) + bitmap = prepare_bitmap_git_for_source(packed); + if (bitmap) { + struct bitmapped_for_each_object_data bitmap_data = { + .packed = packed, + .request = request, + .opts = opts, + .cb = cb, + .cb_data = cb_data, + }; + + ret = for_each_bitmapped_object(bitmap, opts->filter, + bitmapped_for_each_object, + &bitmap_data); + if (ret) + goto out; + } + packed->skip_mru_updates = true; for (e = packfile_store_get_packs(packed); e; e = e->next) { @@ -341,6 +396,13 @@ static int odb_source_packed_for_each_object(struct odb_source *source, if (should_exclude_pack(p, opts->flags)) continue; + /* + * Objects covered by the bitmap have already been yielded + * above; skip them here to avoid duplicates. + */ + if (bitmap && bitmap_index_contains_pack(bitmap, p)) + continue; + if (open_pack_index(p)) { pack_errors = 1; continue; @@ -356,6 +418,7 @@ static int odb_source_packed_for_each_object(struct odb_source *source, out: packed->skip_mru_updates = false; + free_bitmap_index(bitmap); if (!ret && pack_errors) ret = -1; @@ -545,7 +608,8 @@ static int odb_source_packed_write_object_stream(struct odb_source *source UNUSE } static int odb_source_packed_begin_transaction(struct odb_source *source UNUSED, - struct odb_transaction **out UNUSED) + struct odb_transaction **out UNUSED, + enum odb_transaction_flags flags UNUSED) { return error("packed backend cannot begin transactions"); } diff --git a/odb/source.h b/odb/source.h index efeff7ed12adbb..cd63dba91f4e2f 100644 --- a/odb/source.h +++ b/odb/source.h @@ -3,6 +3,7 @@ #include "object.h" #include "odb.h" +#include "odb/transaction.h" enum odb_source_type { /* @@ -232,7 +233,8 @@ struct odb_source { * negative error code otherwise. */ int (*begin_transaction)(struct odb_source *source, - struct odb_transaction **out); + struct odb_transaction **out, + enum odb_transaction_flags flags); /* * This callback is expected to read the list of alternate object @@ -472,9 +474,10 @@ static inline int odb_source_write_alternate(struct odb_source *source, * Returns 0 on success, a negative error code otherwise. */ static inline int odb_source_begin_transaction(struct odb_source *source, - struct odb_transaction **out) + struct odb_transaction **out, + enum odb_transaction_flags flags) { - return source->begin_transaction(source, out); + return source->begin_transaction(source, out, flags); } #endif diff --git a/odb/transaction.c b/odb/transaction.c index b16e07aebfc5ac..dab7da6a9a4f55 100644 --- a/odb/transaction.c +++ b/odb/transaction.c @@ -1,30 +1,41 @@ #include "git-compat-util.h" +#include "gettext.h" #include "odb/source.h" #include "odb/transaction.h" -struct odb_transaction *odb_transaction_begin(struct object_database *odb) +int odb_transaction_begin(struct object_database *odb, + struct odb_transaction **out, + enum odb_transaction_flags flags) { + int ret; + if (odb->transaction) - return NULL; + return error(_("object database transaction already pending")); - odb_source_begin_transaction(odb->sources, &odb->transaction); + ret = odb_source_begin_transaction(odb->sources, out, flags); + if (!ret) + odb->transaction = *out; - return odb->transaction; + return ret; } -void odb_transaction_commit(struct odb_transaction *transaction) +int odb_transaction_commit(struct odb_transaction *transaction) { + int ret; + if (!transaction) - return; + return 0; /* * Ensure the transaction ending matches the pending transaction. */ ASSERT(transaction == transaction->source->odb->transaction); - transaction->commit(transaction); + ret = transaction->commit(transaction); transaction->source->odb->transaction = NULL; free(transaction); + + return ret; } int odb_transaction_write_object_stream(struct odb_transaction *transaction, @@ -33,3 +44,11 @@ int odb_transaction_write_object_stream(struct odb_transaction *transaction, { return transaction->write_object_stream(transaction, stream, len, oid); } + +int odb_transaction_env(struct odb_transaction *transaction, struct strvec *env) +{ + if (!transaction) + return 0; + + return transaction->env(transaction, env); +} diff --git a/odb/transaction.h b/odb/transaction.h index 854fda06f576e4..4cb2eafcbf08f5 100644 --- a/odb/transaction.h +++ b/odb/transaction.h @@ -1,8 +1,8 @@ #ifndef ODB_TRANSACTION_H #define ODB_TRANSACTION_H +#include "gettext.h" #include "odb.h" -#include "odb/source.h" /* * A transaction may be started for an object database prior to writing new @@ -16,8 +16,11 @@ struct odb_transaction { /* The ODB source the transaction is opened against. */ struct odb_source *source; - /* The ODB source specific callback invoked to commit a transaction. */ - void (*commit)(struct odb_transaction *transaction); + /* + * The ODB source specific callback invoked to commit a transaction. + * Returns 0 on success, a negative error code otherwise. + */ + int (*commit)(struct odb_transaction *transaction); /* * This callback is expected to write the given object stream into @@ -30,20 +33,47 @@ struct odb_transaction { int (*write_object_stream)(struct odb_transaction *transaction, struct odb_write_stream *stream, size_t len, struct object_id *oid); + + /* + * This callback is expected to populate the provided strvec with the + * environment variables that a child process should inherit so that its + * object writes participate in the transaction. Returns 0 on success, a + * negative error code otherwise. + */ + int (*env)(struct odb_transaction *transaction, struct strvec *env); +}; + +/* Flags used to configure an ODB transaction. */ +enum odb_transaction_flags { + /* Configures the transaction for use with git-receive-pack(1). */ + ODB_TRANSACTION_RECEIVE = (1 << 0), }; /* - * Starts an ODB transaction. Subsequent objects are written to the transaction - * and not committed until odb_transaction_commit() is invoked on the - * transaction. If the ODB already has a pending transaction, NULL is returned. + * Starts an ODB transaction and returns it via `out`. Subsequent objects are + * written to the transaction and not committed until odb_transaction_commit() + * is invoked on the transaction. Returns 0 on success and a negative value on + * error. Note that it is considered an error to start a new transaction if the + * ODB already has an inflight transaction pending. */ -struct odb_transaction *odb_transaction_begin(struct object_database *odb); +int odb_transaction_begin(struct object_database *odb, + struct odb_transaction **out, + enum odb_transaction_flags flags); + +static inline void odb_transaction_begin_or_die(struct object_database *odb, + struct odb_transaction **out, + enum odb_transaction_flags flags) +{ + if (odb_transaction_begin(odb, out, flags)) + die(_("failed to start ODB transaction")); +} /* - * Commits an ODB transaction making the written objects visible. If the - * specified transaction is NULL, the function is a no-op. + * Commits an ODB transaction making the written objects visible. Returns 0 on + * success, a negative error code otherwise. Note that, if the specified + * transaction is NULL, the function is a no-op and no error is returned. */ -void odb_transaction_commit(struct odb_transaction *transaction); +int odb_transaction_commit(struct odb_transaction *transaction); /* * Writes the object in the provided stream into the transaction. The resulting @@ -54,4 +84,13 @@ int odb_transaction_write_object_stream(struct odb_transaction *transaction, struct odb_write_stream *stream, size_t len, struct object_id *oid); +/* + * Populates the provided strvec with the environment variables that a child + * process should inherit so that its object writes participate in the + * transaction, suitable for using via child_process.env. Returns 0 on success, + * a negative error code otherwise. Note that, if the specified transaction is + * NULL, the function is a no-op and no error is returned. + */ +int odb_transaction_env(struct odb_transaction *transaction, struct strvec *env); + #endif diff --git a/pack-bitmap.c b/pack-bitmap.c index 1fc3e62d5e154d..d8dc4ae8d1633c 100644 --- a/pack-bitmap.c +++ b/pack-bitmap.c @@ -460,8 +460,8 @@ char *pack_bitmap_filename(struct packed_git *p) return xstrfmt("%.*s.bitmap", (int)len, p->pack_name); } -static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, - struct multi_pack_index *midx) +static int open_midx_bitmap(struct bitmap_index *bitmap_git, + struct multi_pack_index *midx) { struct stat st; char *bitmap_name = midx_bitmap_filename(midx); @@ -543,7 +543,7 @@ static int open_midx_bitmap_1(struct bitmap_index *bitmap_git, return -1; } -static int open_pack_bitmap_1(struct bitmap_index *bitmap_git, struct packed_git *packfile) +static int open_pack_bitmap(struct bitmap_index *bitmap_git, struct packed_git *packfile) { int fd; struct stat st; @@ -607,7 +607,7 @@ static int load_reverse_index(struct repository *r, struct bitmap_index *bitmap_ /* * The multi-pack-index's .rev file is already loaded via - * open_pack_bitmap_1(). + * open_pack_bitmap(). * * But we still need to open the individual pack .rev files, * since we will need to make use of them in pack-objects. @@ -684,60 +684,53 @@ static int load_bitmap(struct repository *r, struct bitmap_index *bitmap_git, return 0; } -static int open_pack_bitmap(struct repository *r, - struct bitmap_index *bitmap_git) +static int open_bitmap_for_source(struct odb_source_packed *source, + struct bitmap_index *bitmap_git) { - struct packed_git *p; - int ret = -1; + struct multi_pack_index *midx = get_multi_pack_index(source); + struct packfile_list_entry *e; + bool found = false; - repo_for_each_pack(r, p) { - if (open_pack_bitmap_1(bitmap_git, p) == 0) { - ret = 0; - /* - * The only reason to keep looking is to report - * duplicates. - */ - if (!trace2_is_enabled()) - break; - } + if (midx && !open_midx_bitmap(bitmap_git, midx)) + found = true; + + for (e = packfile_store_get_packs(source); e; e = e->next) { + /* + * When tracing is enabled we want to keep looking to report + * duplicates even if we have already found a bitmap. + */ + if (found && !trace2_is_enabled()) + break; + + if (!open_pack_bitmap(bitmap_git, e->pack)) + found = true; } - return ret; + return found ? 0 : -1; } -static int open_midx_bitmap(struct repository *r, - struct bitmap_index *bitmap_git) +static int open_bitmap(struct repository *r, + struct bitmap_index *bitmap_git) { struct odb_source *source; - int ret = -1; + bool found = false; assert(!bitmap_git->map); odb_prepare_alternates(r->objects); for (source = r->objects->sources; source; source = source->next) { struct odb_source_files *files = odb_source_files_downcast(source); - struct multi_pack_index *midx = get_multi_pack_index(files->packed); - if (midx && !open_midx_bitmap_1(bitmap_git, midx)) - ret = 0; - } - return ret; -} -static int open_bitmap(struct repository *r, - struct bitmap_index *bitmap_git) -{ - int found; - - assert(!bitmap_git->map); + if (!open_bitmap_for_source(files->packed, bitmap_git)) + found = true; - found = !open_midx_bitmap(r, bitmap_git); - - /* - * these will all be skipped if we opened a midx bitmap; but run it - * anyway if tracing is enabled to report the duplicates - */ - if (!found || trace2_is_enabled()) - found |= !open_pack_bitmap(r, bitmap_git); + /* + * The only reason to keep looking after having found a bitmap + * is to report duplicates. + */ + if (found && !trace2_is_enabled()) + break; + } return found ? 0 : -1; } @@ -757,7 +750,19 @@ struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx) { struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git)); - if (!open_midx_bitmap_1(bitmap_git, midx)) + if (!open_midx_bitmap(bitmap_git, midx)) + return bitmap_git; + + free_bitmap_index(bitmap_git); + return NULL; +} + +struct bitmap_index *prepare_bitmap_git_for_source(struct odb_source_packed *source) +{ + struct bitmap_index *bitmap_git = xcalloc(1, sizeof(*bitmap_git)); + + if (!open_bitmap_for_source(source, bitmap_git) && + !load_bitmap(source->base.odb->repo, bitmap_git, 0)) return bitmap_git; free_bitmap_index(bitmap_git); @@ -1699,7 +1704,7 @@ static void init_type_iterator(struct ewah_or_iterator *it, } } -static void show_objects_for_type( +static int show_objects_for_type( struct bitmap_index *bitmap_git, struct bitmap *objects, enum object_type object_type, @@ -1708,6 +1713,7 @@ static void show_objects_for_type( { size_t i = 0; uint32_t offset; + int ret; struct ewah_or_iterator it; eword_t filter; @@ -1753,11 +1759,17 @@ static void show_objects_for_type( hash = bitmap_name_hash(bitmap_git, index_pos); - show_reach(&oid, object_type, 0, hash, pack, ofs, payload); + ret = show_reach(&oid, object_type, 0, hash, pack, ofs, payload); + if (ret) + goto out; } } + ret = 0; + +out: ewah_or_iterator_release(&it); + return ret; } static int in_bitmapped_pack(struct bitmap_index *bitmap_git, @@ -1980,7 +1992,7 @@ static void filter_bitmap_object_type(struct bitmap_index *bitmap_git, static int filter_bitmap(struct bitmap_index *bitmap_git, struct object_list *tip_objects, struct bitmap *to_filter, - struct list_objects_filter_options *filter) + const struct list_objects_filter_options *filter) { if (!filter || filter->choice == LOFC_DISABLED) return 0; @@ -2031,12 +2043,11 @@ static int filter_bitmap(struct bitmap_index *bitmap_git, return -1; } -static int can_filter_bitmap(struct list_objects_filter_options *filter) +bool can_filter_bitmap(const struct list_objects_filter_options *filter) { return !filter_bitmap(NULL, NULL, NULL, filter); } - static void filter_packed_objects_from_bitmap(struct bitmap_index *bitmap_git, struct bitmap *result) { @@ -2062,10 +2073,16 @@ static void filter_packed_objects_from_bitmap(struct bitmap_index *bitmap_git, } int for_each_bitmapped_object(struct bitmap_index *bitmap_git, - struct list_objects_filter_options *filter, + const struct list_objects_filter_options *filter, show_reachable_fn show_reach, void *payload) { + const enum object_type types[] = { + OBJ_COMMIT, + OBJ_TREE, + OBJ_BLOB, + OBJ_TAG, + }; struct bitmap *filtered_bitmap = NULL; uint32_t objects_nr; size_t full_word_count; @@ -2090,14 +2107,12 @@ int for_each_bitmapped_object(struct bitmap_index *bitmap_git, goto out; } - show_objects_for_type(bitmap_git, filtered_bitmap, - OBJ_COMMIT, show_reach, payload); - show_objects_for_type(bitmap_git, filtered_bitmap, - OBJ_TREE, show_reach, payload); - show_objects_for_type(bitmap_git, filtered_bitmap, - OBJ_BLOB, show_reach, payload); - show_objects_for_type(bitmap_git, filtered_bitmap, - OBJ_TAG, show_reach, payload); + for (size_t i = 0; i < ARRAY_SIZE(types); i++) { + ret = show_objects_for_type(bitmap_git, filtered_bitmap, + types[i], show_reach, payload); + if (ret) + goto out; + } ret = 0; out: diff --git a/pack-bitmap.h b/pack-bitmap.h index 19a86554579f7c..1385027c1ff5fa 100644 --- a/pack-bitmap.h +++ b/pack-bitmap.h @@ -9,6 +9,7 @@ #include "string-list.h" struct commit; +struct odb_source_packed; struct repository; struct rev_info; @@ -68,6 +69,7 @@ struct bitmapped_pack { struct bitmap_index *prepare_bitmap_git(struct repository *r); struct bitmap_index *prepare_midx_bitmap_git(struct multi_pack_index *midx); +struct bitmap_index *prepare_bitmap_git_for_source(struct odb_source_packed *source); /* * Given a bitmap index, determine whether it contains the pack either directly @@ -90,13 +92,17 @@ int test_bitmap_pseudo_merge_objects(struct repository *r, uint32_t n); struct list_objects_filter_options; +/* Check whether the filter can be computed via the bitmap. */ +bool can_filter_bitmap(const struct list_objects_filter_options *filter); + /* * Filter bitmapped objects and iterate through all resulting objects, * executing `show_reach` for each of them. Returns `-1` in case the filter is - * not supported, `0` otherwise. + * not supported, `0` otherwise. Aborts iteration and bubbles up the return + * value in case `show_reach()` returns non-zero. */ int for_each_bitmapped_object(struct bitmap_index *bitmap_git, - struct list_objects_filter_options *filter, + const struct list_objects_filter_options *filter, show_reachable_fn show_reach, void *payload); diff --git a/read-cache.c b/read-cache.c index 797fdfc07dfbfe..38b55323dd739a 100644 --- a/read-cache.c +++ b/read-cache.c @@ -4015,6 +4015,7 @@ int add_files_to_cache(struct repository *repo, const char *prefix, const struct pathspec *pathspec, char *ps_matched, int include_sparse, int flags, int ignored_too ) { + int inflight = !!repo->objects->transaction; struct odb_transaction *transaction; struct update_callback_data data; struct rev_info rev; @@ -4045,9 +4046,11 @@ int add_files_to_cache(struct repository *repo, const char *prefix, * This function is invoked from commands other than 'add', which * may not have their own transaction active. */ - transaction = odb_transaction_begin(repo->objects); + if (!inflight) + odb_transaction_begin_or_die(repo->objects, &transaction, 0); run_diff_files(&rev, DIFF_RACY_IS_MODIFIED); - odb_transaction_commit(transaction); + if (!inflight) + odb_transaction_commit(transaction); release_revisions(&rev); return !!data.add_errors; diff --git a/revision.c b/revision.c index 8425f67cbeab0b..fec5472580e058 100644 --- a/revision.c +++ b/revision.c @@ -4424,6 +4424,7 @@ static struct commit *get_revision_1(struct rev_info *revs) switch (mode) { case REV_WALK_REFLOG: + case REV_WALK_NO_WALK: try_to_simplify_commit(revs, commit); break; case REV_WALK_TOPO: @@ -4437,7 +4438,6 @@ static struct commit *get_revision_1(struct rev_info *revs) oid_to_hex(&commit->object.oid)); } break; - case REV_WALK_NO_WALK: case REV_WALK_LIMITED: break; } diff --git a/strbuf.c b/strbuf.c index 764b629927f0ce..44955669e8c504 100644 --- a/strbuf.c +++ b/strbuf.c @@ -646,8 +646,6 @@ int strbuf_getwholeline(struct strbuf *sb, FILE *fp, int term) if (feof(fp)) return EOF; - strbuf_reset(sb); - /* Translate slopbuf to NULL, as we cannot call realloc on it */ if (!sb->alloc) sb->buf = NULL; diff --git a/t/t0450/adoc-help-mismatches b/t/t0450/adoc-help-mismatches index e8d6c13ccd0333..c4a55ff4e35a4f 100644 --- a/t/t0450/adoc-help-mismatches +++ b/t/t0450/adoc-help-mismatches @@ -12,7 +12,6 @@ column credential credential-cache credential-store -fast-export fast-import fetch-pack fmt-merge-msg diff --git a/t/t1100-commit-tree-options.sh b/t/t1100-commit-tree-options.sh index ae66ba5babf347..3e888909f58d30 100755 --- a/t/t1100-commit-tree-options.sh +++ b/t/t1100-commit-tree-options.sh @@ -14,37 +14,35 @@ Also make sure that command line parser understands the normal . ./test-lib.sh -cat >expected < 1117148400 +0000 -committer Committer Name 1117150200 +0000 - -comment text -EOF - -test_expect_success \ - 'test preparation: write empty tree' \ - 'git write-tree >treeid' - -test_expect_success \ - 'construct commit' \ - 'echo comment text | - GIT_AUTHOR_NAME="Author Name" \ - GIT_AUTHOR_EMAIL="author@email" \ - GIT_AUTHOR_DATE="2005-05-26 23:00" \ - GIT_COMMITTER_NAME="Committer Name" \ - GIT_COMMITTER_EMAIL="committer@email" \ - GIT_COMMITTER_DATE="2005-05-26 23:30" \ - TZ=GMT git commit-tree $(cat treeid) >commitid 2>/dev/null' - -test_expect_success \ - 'read commit' \ - 'git cat-file commit $(cat commitid) >commit' - -test_expect_success \ - 'compare commit' \ - 'test_cmp expected commit' +test_expect_success 'test preparation: write empty tree' ' + cat >expected <<-EOF && + tree $EMPTY_TREE + author Author Name 1117148400 +0000 + committer Committer Name 1117150200 +0000 + + comment text + EOF + git write-tree >treeid +' + +test_expect_success 'construct commit' ' + echo comment text | + GIT_AUTHOR_NAME="Author Name" \ + GIT_AUTHOR_EMAIL="author@email" \ + GIT_AUTHOR_DATE="2005-05-26 23:00" \ + GIT_COMMITTER_NAME="Committer Name" \ + GIT_COMMITTER_EMAIL="committer@email" \ + GIT_COMMITTER_DATE="2005-05-26 23:30" \ + TZ=GMT git commit-tree $(cat treeid) >commitid 2>/dev/null +' +test_expect_success 'read commit' ' + git cat-file commit $(cat commitid) >commit +' + +test_expect_success 'compare commit' ' + test_cmp expected commit +' test_expect_success 'flags and then non flags' ' test_tick && diff --git a/t/t1305-config-include.sh b/t/t1305-config-include.sh index f6115269f9ede0..507b8ebd061b00 100755 --- a/t/t1305-config-include.sh +++ b/t/t1305-config-include.sh @@ -396,4 +396,132 @@ test_expect_success 'onbranch without repository but explicit nonexistent Git di test_must_fail nongit git --git-dir=nonexistent config get foo.bar ' +# worktree: conditional include tests + +test_expect_success 'conditional include, worktree bare repo' ' + git init --bare wt-bare && + ( + cd wt-bare && + echo "[includeIf \"worktree:/\"]path=bar-bare" >>config && + echo "[test]wtbare=1" >bar-bare && + test_must_fail git config test.wtbare + ) +' + +test_expect_success 'conditional include, worktree multiple worktrees' ' + git init wt-multi && + ( + cd wt-multi && + test_commit initial && + git worktree add -b linked-branch ../wt-linked HEAD && + git worktree add -b prefix-branch ../wt-prefix/linked HEAD + ) && + wt_main="$(cd wt-multi && pwd)" && + wt_linked="$(cd wt-linked && pwd)" && + wt_prefix_parent="$(cd wt-prefix && pwd)" && + cat >>wt-multi/.git/config <<-EOF && + [includeIf "worktree:$wt_main"] + path = main-config + [includeIf "worktree:$wt_linked"] + path = linked-config + [includeIf "worktree:$wt_prefix_parent/"] + path = prefix-config + EOF + echo "[test]mainvar=main" >wt-multi/.git/main-config && + echo "[test]linkedvar=linked" >wt-multi/.git/linked-config && + echo "[test]prefixvar=prefix" >wt-multi/.git/prefix-config && + echo main >expect && + git -C wt-multi config test.mainvar >actual && + test_cmp expect actual && + test_must_fail git -C wt-multi config test.linkedvar && + test_must_fail git -C wt-multi config test.prefixvar && + echo linked >expect && + git -C wt-linked config test.linkedvar >actual && + test_cmp expect actual && + test_must_fail git -C wt-linked config test.mainvar && + test_must_fail git -C wt-linked config test.prefixvar && + echo prefix >expect && + git -C wt-prefix/linked config test.prefixvar >actual && + test_cmp expect actual && + test_must_fail git -C wt-prefix/linked config test.mainvar && + test_must_fail git -C wt-prefix/linked config test.linkedvar +' + +test_expect_success SYMLINKS 'conditional include, worktree resolves symlinks' ' + mkdir real-wt && + ln -s real-wt link-wt && + git init link-wt/repo && + ( + cd link-wt/repo && + # repo->worktree resolves symlinks, so use real path in pattern + echo "[includeIf \"worktree:**/real-wt/repo\"]path=bar-link" >>.git/config && + echo "[test]wtlink=2" >.git/bar-link && + echo 2 >expect && + git config test.wtlink >actual && + test_cmp expect actual + ) +' + +test_expect_success !CASE_INSENSITIVE_FS 'conditional include, worktree, case sensitive' ' + git init wt-case && + ( + cd wt-case && + test_commit initial && + wt_path="$(pwd)" && + wt_upper=$(echo "$wt_path" | tr a-z A-Z) && + echo "[includeIf \"worktree:$wt_upper\"]path=case-inc" >>.git/config && + echo "[test]wtcase=1" >.git/case-inc && + test_must_fail git config test.wtcase + ) +' + +test_expect_success 'conditional include, worktree, icase' ' + git init wt-icase && + ( + cd wt-icase && + test_commit initial && + wt_path="$(pwd)" && + wt_upper=$(echo "$wt_path" | tr a-z A-Z) && + echo "[includeIf \"worktree/i:$wt_upper\"]path=icase-inc" >>.git/config && + echo "[test]wticase=1" >.git/icase-inc && + echo 1 >expect && + git config test.wticase >actual && + test_cmp expect actual + ) +' + +# The "worktree" condition cannot match during early config reading +# because the repository object is not yet fully initialized and +# repo_get_work_tree() returns NULL. +test_expect_success 'conditional include, worktree does not match in early config' ' + git init wt-early && + ( + cd wt-early && + test_commit initial && + wt_path="$(pwd)" && + echo "[includeIf \"worktree:$wt_path\"]path=early-inc" >>.git/config && + echo "[test]wtearly=1" >.git/early-inc && + test-tool config read_early_config test.wtearly >actual && + test_must_be_empty actual + ) +' + +# Use a loose pattern so the "present in non-worktree cases" check works +# for Unix-style absolute paths and Windows paths like D:/a/git/... +test_expect_success 'conditional include, worktree without repository' ' + test_when_finished "rm -f .gitconfig config.inc" && + git config set -f .gitconfig "includeIf.worktree:**.path" config.inc && + git config set -f config.inc foo.bar baz && + git config get foo.bar && + test_must_fail nongit git config get foo.bar +' + +test_expect_success 'conditional include, worktree without repository but explicit nonexistent Git directory' ' + test_when_finished "rm -f .gitconfig config.inc" && + git config set -f .gitconfig "includeIf.worktree:**.path" config.inc && + git config set -f config.inc foo.bar baz && + git config get foo.bar && + test_must_fail nongit git --git-dir=nonexistent config get foo.bar +' + test_done diff --git a/t/t6017-rev-list-stdin.sh b/t/t6017-rev-list-stdin.sh index 4821b90e7479ad..32284f183176a7 100755 --- a/t/t6017-rev-list-stdin.sh +++ b/t/t6017-rev-list-stdin.sh @@ -148,4 +148,22 @@ test_expect_success '--not via stdin does not influence revisions from command l test_cmp expect actual ' +test_expect_success '--no-walk filters by path (single commit, match)' ' + git rev-parse side-1 >expect && + git rev-list --no-walk side-1 -- file-1 >actual && + test_cmp expect actual +' + +test_expect_success '--no-walk filters by path (single commit, no match)' ' + git rev-list --no-walk side-2 -- file-1 >actual && + test_must_be_empty actual +' + +test_expect_success '--no-walk with pathspec exclusion' ' + git rev-parse side-3 side-2 >expect && + git rev-parse side-1 side-2 side-3 >input && + git rev-list --stdin --no-walk -- ":!file-1" actual && + test_cmp expect actual +' + test_done diff --git a/t/t7614-merge-signoff.sh b/t/t7614-merge-signoff.sh index fee258d4f0c2c0..e58bf07b7a4540 100755 --- a/t/t7614-merge-signoff.sh +++ b/t/t7614-merge-signoff.sh @@ -45,7 +45,8 @@ test_expect_success 'git merge --signoff adds a sign-off line' ' test_commit main-branch-2 file2 2 && git checkout other-branch && git merge main --signoff --no-edit && - git cat-file commit HEAD | sed -e "1,/^\$/d" >actual && + git cat-file commit HEAD >commit && + sed -e "1,/^\$/d" commit >actual && test_cmp expected-signed actual ' @@ -55,7 +56,8 @@ test_expect_success 'git merge does not add a sign-off line' ' test_commit main-branch-3 file3 3 && git checkout other-branch && git merge main --no-edit && - git cat-file commit HEAD | sed -e "1,/^\$/d" >actual && + git cat-file commit HEAD >commit && + sed -e "1,/^\$/d" commit >actual && test_cmp expected-unsigned actual ' @@ -65,7 +67,8 @@ test_expect_success 'git merge --no-signoff flag cancels --signoff flag' ' test_commit main-branch-4 file4 4 && git checkout other-branch && git merge main --no-edit --signoff --no-signoff && - git cat-file commit HEAD | sed -e "1,/^\$/d" >actual && + git cat-file commit HEAD >commit && + sed -e "1,/^\$/d" commit >actual && test_cmp expected-unsigned actual ' diff --git a/t/t9811-git-p4-label-import.sh b/t/t9811-git-p4-label-import.sh index 7614dfbd953efe..ee257011b3c97f 100755 --- a/t/t9811-git-p4-label-import.sh +++ b/t/t9811-git-p4-label-import.sh @@ -62,9 +62,11 @@ test_expect_success 'basic p4 labels' ' cd main && git checkout TAG_F1_ONLY && - ! test -f f2 && + test_path_is_missing f2 && git checkout TAG_WITH\$_SHELL_CHAR && - test -f f1 && test -f f2 && test -f file_with_\$metachar && + test_path_is_file f1 && + test_path_is_file f2 && + test_path_is_file file_with_\$metachar && git show TAG_LONG_LABEL | grep -q "A Label second line" ) @@ -102,11 +104,11 @@ test_expect_success 'two labels on the same changelist' ' git checkout TAG_F1_1 && ls && - test -f f1 && + test_path_is_file f1 && git checkout TAG_F1_2 && ls && - test -f f1 + test_path_is_file f1 ) ' @@ -135,9 +137,9 @@ test_expect_success 'export git tags to p4' ' p4 labels ... | grep LIGHTWEIGHT_TAG && p4 label -o GIT_TAG_1 | grep "tag created in git:xyzzy" && p4 sync ...@GIT_TAG_1 && - ! test -f main/f10 && + test_path_is_missing main/f10 && p4 sync ...@GIT_TAG_2 && - test -f main/f10 + test_path_is_file main/f10 ) ' @@ -168,9 +170,9 @@ test_expect_success 'export git tags to p4 with deletion' ' cd "$cli" && p4 sync ... && p4 sync ...@GIT_TAG_ON_DELETED && - test -f main/deleted_file && + test_path_is_file main/deleted_file && p4 sync ...@GIT_TAG_AFTER_DELETION && - ! test -f main/deleted_file && + test_path_is_missing main/deleted_file && echo "checking label contents" && p4 label -o GIT_TAG_ON_DELETED | grep "tag on deleted file" ) @@ -231,17 +233,25 @@ test_expect_success 'importing labels with missing revisions' ' P4CLIENT=missing-revision && client_view "//depot/missing-revision/... //missing-revision/..." && cd "$cli" && - >f1 && p4 add f1 && p4 submit -d "start" && + >f1 && + p4 add f1 && + p4 submit -d "start" && p4 tag -l TAG_S0 ... && - >f2 && p4 add f2 && p4 submit -d "second" && + >f2 && + p4 add f2 && + p4 submit -d "second" && startrev=$(p4_head_revision //depot/missing-revision/...) && - >f3 && p4 add f3 && p4 submit -d "third" && + >f3 && + p4 add f3 && + p4 submit -d "third" && - p4 edit f2 && date >f2 && p4 submit -d "change" f2 && + p4 edit f2 && + date >f2 && + p4 submit -d "change" f2 && endrev=$(p4_head_revision //depot/missing-revision/...) &&