Skip to content

Commit bd2c42d

Browse files
gitstergitster
committed
Merge branch 'ds/bundle-uri-more' into jch
RFC/ seems to trigger errors in linux-leaks CI job source: <[email protected]> * ds/bundle-uri-more: fetch: add 'refs/bundle/' to log.excludeDecoration bundle-uri: add support for http(s):// and file:// fetch: add --bundle-uri option bundle-uri: create basic file-copy logic remote-curl: add 'get' capability docs: document bundle URI standard
2 parents f018ae2 + b5c70a3 commit bd2c42d

File tree

12 files changed

+833
-1
lines changed

12 files changed

+833
-1
lines changed

‎Documentation/fetch-options.txt‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -317,3 +317,9 @@ endif::git-pull[]
317317
-6::
318318
--ipv6::
319319
Use IPv6 addresses only, ignoring IPv4 addresses.
320+
321+
--bundle-uri=<uri>::
322+
Instead of fetching from a remote, fetch a bundle from the given
323+
`<uri>` and unbundle the data into the local repository. The refs
324+
in the bundle will be stored under the hidden `refs/bundle/*`
325+
namespace.

‎Documentation/git-fetch.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ SYNOPSIS
1313
'git fetch' [<options>] <group>
1414
'git fetch' --multiple [<options>] [(<repository> | <group>)...]
1515
'git fetch' --all [<options>]
16+
'git fetch' --bundle-uri=<uri> [<options>]
1617

1718

1819
DESCRIPTION

‎Documentation/gitremote-helpers.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,9 @@ Supported commands: 'list', 'import'.
168168
Can guarantee that when a clone is requested, the received
169169
pack is self contained and is connected.
170170

171+
'get'::
172+
Can use the 'get' command to download a file from a given URI.
173+
171174
If a helper advertises 'connect', Git will use it if possible and
172175
fall back to another capability if the helper requests so when
173176
connecting (see the 'connect' command under COMMANDS).
@@ -418,6 +421,12 @@ Supported if the helper has the "connect" capability.
418421
+
419422
Supported if the helper has the "stateless-connect" capability.
420423

424+
'get' <uri> <path>::
425+
Downloads the file from the given `<uri>` to the given `<path>`. If
426+
`<path>.temp` exists, then Git assumes that the `.temp` file is a
427+
partial download from a previous attempt and will resume the
428+
download from that position.
429+
421430
If a fatal error occurs, the program writes the error message to
422431
stderr and exits. The caller should expect that a suitable error
423432
message has been printed if the child closes the connection without

‎Documentation/technical/bundle-uri.txt‎

Lines changed: 475 additions & 0 deletions
Large diffs are not rendered by default.

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -899,6 +899,7 @@ LIB_OBJS += blob.o
899899
LIB_OBJS += bloom.o
900900
LIB_OBJS += branch.o
901901
LIB_OBJS += bulk-checkin.o
902+
LIB_OBJS += bundle-uri.o
902903
LIB_OBJS += bundle.o
903904
LIB_OBJS += cache-tree.o
904905
LIB_OBJS += cbtree.o

‎builtin/fetch.c‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
#include "commit-graph.h"
3030
#include "shallow.h"
3131
#include "worktree.h"
32+
#include "bundle-uri.h"
3233

3334
#define FORCED_UPDATES_DELAY_WARNING_IN_MS (10 * 1000)
3435

@@ -37,6 +38,7 @@ static const char * const builtin_fetch_usage[] = {
3738
N_("git fetch [<options>] <group>"),
3839
N_("git fetch --multiple [<options>] [(<repository> | <group>)...]"),
3940
N_("git fetch --all [<options>]"),
41+
N_("git fetch --bundle-uri=<uri> [<options>]"),
4042
NULL
4143
};
4244

@@ -86,6 +88,7 @@ static struct string_list negotiation_tip = STRING_LIST_INIT_NODUP;
8688
static int fetch_write_commit_graph = -1;
8789
static int stdin_refspecs = 0;
8890
static int negotiate_only;
91+
static const char *bundle_uri;
8992

9093
static int git_fetch_config(const char *k, const char *v, void *cb)
9194
{
@@ -224,6 +227,8 @@ static struct option builtin_fetch_options[] = {
224227
N_("write the commit-graph after fetching")),
225228
OPT_BOOL(0, "stdin", &stdin_refspecs,
226229
N_("accept refspecs from stdin")),
230+
OPT_STRING(0, "bundle-uri", &bundle_uri, N_("uri"),
231+
N_("download bundle data from the given URI instead of from a remote")),
227232
OPT_END()
228233
};
229234

@@ -2182,6 +2187,11 @@ int cmd_fetch(int argc, const char **argv, const char *prefix)
21822187
if (dry_run)
21832188
write_fetch_head = 0;
21842189

2190+
if (bundle_uri) {
2191+
result = fetch_bundle_uri(the_repository, bundle_uri);
2192+
goto cleanup;
2193+
}
2194+
21852195
if (all) {
21862196
if (argc == 1)
21872197
die(_("fetch --all does not take a repository argument"));

‎bundle-uri.c‎

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
#include "cache.h"
2+
#include "bundle-uri.h"
3+
#include "bundle.h"
4+
#include "config.h"
5+
#include "object-store.h"
6+
#include "refs.h"
7+
#include "run-command.h"
8+
9+
static void find_temp_filename(struct strbuf *name)
10+
{
11+
int fd;
12+
/*
13+
* Find a temporray filename that is available. This is briefly
14+
* racy, but unlikely to collide.
15+
*/
16+
fd = odb_mkstemp(name, "bundles/tmp_uri_XXXXXX");
17+
if (fd < 0)
18+
die(_("failed to create temporary file"));
19+
close(fd);
20+
unlink(name->buf);
21+
}
22+
23+
static int download_https_uri_to_file(const char *uri, const char *file)
24+
{
25+
int result = 0;
26+
struct child_process cp = CHILD_PROCESS_INIT;
27+
FILE *child_in = NULL, *child_out = NULL;
28+
struct strbuf line = STRBUF_INIT;
29+
int found_get = 0;
30+
31+
strvec_pushl(&cp.args, "git-remote-https", "origin", uri, NULL);
32+
cp.in = -1;
33+
cp.out = -1;
34+
35+
if (start_command(&cp))
36+
return 1;
37+
38+
child_in = fdopen(cp.in, "w");
39+
if (!child_in) {
40+
result = 1;
41+
goto cleanup;
42+
}
43+
44+
child_out = fdopen(cp.out, "r");
45+
if (!child_out) {
46+
result = 1;
47+
goto cleanup;
48+
}
49+
50+
fprintf(child_in, "capabilities\n");
51+
fflush(child_in);
52+
53+
while (!strbuf_getline(&line, child_out)) {
54+
if (!line.len)
55+
break;
56+
if (!strcmp(line.buf, "get"))
57+
found_get = 1;
58+
}
59+
strbuf_release(&line);
60+
61+
if (!found_get) {
62+
result = error(_("insufficient capabilities"));
63+
goto cleanup;
64+
}
65+
66+
fprintf(child_in, "get %s %s\n\n", uri, file);
67+
68+
cleanup:
69+
if (child_in)
70+
fclose(child_in);
71+
if (finish_command(&cp))
72+
return 1;
73+
if (child_out)
74+
fclose(child_out);
75+
return result;
76+
}
77+
78+
static int copy_uri_to_file(const char *uri, const char *file)
79+
{
80+
const char *out;
81+
if (skip_prefix(uri, "https:", &out) ||
82+
skip_prefix(uri, "http:", &out))
83+
return download_https_uri_to_file(uri, file);
84+
85+
if (!skip_prefix(uri, "file://", &out))
86+
out = uri;
87+
88+
/* Copy as a file */
89+
return !!copy_file(file, out, 0);
90+
}
91+
92+
static int unbundle_from_file(struct repository *r, const char *file)
93+
{
94+
int result = 0;
95+
int bundle_fd;
96+
struct bundle_header header = BUNDLE_HEADER_INIT;
97+
struct strvec extra_index_pack_args = STRVEC_INIT;
98+
struct string_list_item *refname;
99+
struct strbuf bundle_ref = STRBUF_INIT;
100+
size_t bundle_prefix_len;
101+
102+
if ((bundle_fd = read_bundle_header(file, &header)) < 0)
103+
return 1;
104+
105+
result = unbundle(r, &header, bundle_fd, &extra_index_pack_args);
106+
107+
/*
108+
* Convert all refs/heads/ from the bundle into refs/bundles/
109+
* in the local repository.
110+
*/
111+
strbuf_addstr(&bundle_ref, "refs/bundles/");
112+
bundle_prefix_len = bundle_ref.len;
113+
114+
for_each_string_list_item(refname, &header.references) {
115+
struct object_id *oid = refname->util;
116+
struct object_id old_oid;
117+
const char *branch_name;
118+
int has_old;
119+
120+
if (!skip_prefix(refname->string, "refs/heads/", &branch_name))
121+
continue;
122+
123+
strbuf_setlen(&bundle_ref, bundle_prefix_len);
124+
strbuf_addstr(&bundle_ref, branch_name);
125+
126+
has_old = !read_ref(bundle_ref.buf, &old_oid);
127+
update_ref("fetched bundle", bundle_ref.buf, oid,
128+
has_old ? &old_oid : NULL,
129+
REF_SKIP_OID_VERIFICATION,
130+
UPDATE_REFS_MSG_ON_ERR);
131+
}
132+
133+
bundle_header_release(&header);
134+
return result;
135+
}
136+
137+
int fetch_bundle_uri(struct repository *r, const char *uri)
138+
{
139+
int result = 0;
140+
struct strbuf filename = STRBUF_INIT;
141+
142+
find_temp_filename(&filename);
143+
if ((result = copy_uri_to_file(uri, filename.buf))) {
144+
error(_("failed to download bundle from URI '%s'"), uri);
145+
goto cleanup;
146+
}
147+
148+
if ((result = !is_bundle(filename.buf, 0))) {
149+
error(_("file at URI '%s' is not a bundle"), uri);
150+
goto cleanup;
151+
}
152+
153+
if ((result = unbundle_from_file(r, filename.buf)))
154+
goto cleanup;
155+
156+
git_config_set_multivar_gently("log.excludedecoration",
157+
"refs/bundle/",
158+
"refs/bundle/",
159+
CONFIG_FLAGS_FIXED_VALUE |
160+
CONFIG_FLAGS_MULTI_REPLACE);
161+
162+
cleanup:
163+
unlink(filename.buf);
164+
strbuf_release(&filename);
165+
return result;
166+
}

‎bundle-uri.h‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#ifndef BUNDLE_URI_H
2+
#define BUNDLE_URI_H
3+
4+
struct repository;
5+
6+
/**
7+
* Fetch data from the given 'uri' and unbundle the bundle data found
8+
* based on that information.
9+
*
10+
* Returns non-zero if no bundle information is found at the given 'uri'.
11+
*/
12+
int fetch_bundle_uri(struct repository *r, const char *uri);
13+
14+
#endif

‎remote-curl.c‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1276,6 +1276,34 @@ static void parse_fetch(struct strbuf *buf)
12761276
strbuf_reset(buf);
12771277
}
12781278

1279+
static void parse_get(struct strbuf *buf)
1280+
{
1281+
struct http_get_options opts = { 0 };
1282+
struct strbuf url = STRBUF_INIT;
1283+
struct strbuf path = STRBUF_INIT;
1284+
const char *p, *space;
1285+
1286+
if (!skip_prefix(buf->buf, "get ", &p))
1287+
die(_("http transport does not support %s"), buf->buf);
1288+
1289+
space = strchr(p, ' ');
1290+
1291+
if (!space)
1292+
die(_("protocol error: expected '<url> <path>', missing space"));
1293+
1294+
strbuf_add(&url, p, space - p);
1295+
strbuf_addstr(&path, space + 1);
1296+
1297+
if (http_get_file(url.buf, path.buf, &opts))
1298+
die(_("failed to download file at URL '%s'"), url.buf);
1299+
1300+
strbuf_release(&url);
1301+
strbuf_release(&path);
1302+
printf("\n");
1303+
fflush(stdout);
1304+
strbuf_reset(buf);
1305+
}
1306+
12791307
static int push_dav(int nr_spec, const char **specs)
12801308
{
12811309
struct child_process child = CHILD_PROCESS_INIT;
@@ -1549,9 +1577,14 @@ int cmd_main(int argc, const char **argv)
15491577
printf("unsupported\n");
15501578
fflush(stdout);
15511579

1580+
} else if (skip_prefix(buf.buf, "get ", &arg)) {
1581+
parse_get(&buf);
1582+
fflush(stdout);
1583+
15521584
} else if (!strcmp(buf.buf, "capabilities")) {
15531585
printf("stateless-connect\n");
15541586
printf("fetch\n");
1587+
printf("get\n");
15551588
printf("option\n");
15561589
printf("push\n");
15571590
printf("check-connectivity\n");

‎t/t5557-http-get.sh‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
#!/bin/sh
2+
3+
test_description='test downloading a file by URL'
4+
5+
. ./test-lib.sh
6+
7+
. "$TEST_DIRECTORY"/lib-httpd.sh
8+
start_httpd
9+
10+
test_expect_success 'get by URL: 404' '
11+
url="$HTTPD_URL/none.txt" &&
12+
cat >input <<-EOF &&
13+
capabilities
14+
get $url file1
15+
EOF
16+
17+
test_must_fail git remote-http $url $url <input 2>err &&
18+
test_path_is_missing file1 &&
19+
grep "failed to download file at URL" err &&
20+
rm file1.temp
21+
'
22+
23+
test_expect_success 'get by URL: 200' '
24+
echo data >"$HTTPD_DOCUMENT_ROOT_PATH/exists.txt" &&
25+
26+
url="$HTTPD_URL/exists.txt" &&
27+
cat >input <<-EOF &&
28+
capabilities
29+
get $url file2
30+
31+
EOF
32+
33+
GIT_TRACE2_PERF=1 git remote-http $url $url <input &&
34+
test_cmp "$HTTPD_DOCUMENT_ROOT_PATH/exists.txt" file2
35+
'
36+
37+
test_done

0 commit comments

Comments
 (0)