Skip to content

Commit 5b08b8f

Browse files
alexcrichtonalexcrichton
committed
Support vendoring git repositories
Currently the vendoring support in Cargo primarily only allows replacing registry sources, e.g. crates.io. Other networked sources of code, such as git repositories, cannot currently be replaced. The purpose of this commit is to support vendoring of git dependencies to eventually have support implemented in the `cargo-vendor` subcommand. Support for vendoring git repositories required a few subtle changes: * First and foremost, configuration for source replacement of a git repository was added. This looks similar to the `Cargo.toml` configuration of a git source. * The restriction around checksum providing sources was relaxed. If a replacement source provides checksums but the replaced source doesn't then that's now considered ok unlike it being an error before. * Lock files can be generated for crates.io crates against vendored sources, but lock files cannot be generated against git sources. A lock file must previously exist to make use of a vendored git source. * The `package` field of `.cargo-checksum.json` is now optional, and it is intended to be omitted for git sources that are vendored.
1 parent 4a7cae5 commit 5b08b8f

File tree

10 files changed

+248
-22
lines changed

10 files changed

+248
-22
lines changed

‎src/cargo/core/registry.rs

+29-5
Original file line numberDiff line numberDiff line change
@@ -26,11 +26,11 @@ pub trait Registry {
2626

2727
/// Returns whether or not this registry will return summaries with
2828
/// checksums listed.
29-
///
30-
/// By default, registries do not support checksums.
31-
fn supports_checksums(&self) -> bool {
32-
false
33-
}
29+
fn supports_checksums(&self) -> bool;
30+
31+
/// Returns whether or not this registry will return summaries with
32+
/// the `precise` field in the source id listed.
33+
fn requires_precise(&self) -> bool;
3434
}
3535

3636
impl<'a, T: ?Sized + Registry + 'a> Registry for Box<T> {
@@ -39,6 +39,14 @@ impl<'a, T: ?Sized + Registry + 'a> Registry for Box<T> {
3939
f: &mut FnMut(Summary)) -> CargoResult<()> {
4040
(**self).query(dep, f)
4141
}
42+
43+
fn supports_checksums(&self) -> bool {
44+
(**self).supports_checksums()
45+
}
46+
47+
fn requires_precise(&self) -> bool {
48+
(**self).requires_precise()
49+
}
4250
}
4351

4452
/// This structure represents a registry of known packages. It internally
@@ -415,6 +423,14 @@ impl<'cfg> Registry for PackageRegistry<'cfg> {
415423
f(self.lock(override_summary));
416424
Ok(())
417425
}
426+
427+
fn supports_checksums(&self) -> bool {
428+
false
429+
}
430+
431+
fn requires_precise(&self) -> bool {
432+
false
433+
}
418434
}
419435

420436
fn lock(locked: &LockedMap,
@@ -579,5 +595,13 @@ pub mod test {
579595
Ok(())
580596
}
581597
}
598+
599+
fn supports_checksums(&self) -> bool {
600+
false
601+
}
602+
603+
fn requires_precise(&self) -> bool {
604+
false
605+
}
582606
}
583607
}

‎src/cargo/sources/config.rs

+42-10
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ use std::path::{Path, PathBuf};
99

1010
use url::Url;
1111

12-
use core::{Source, SourceId};
12+
use core::{Source, SourceId, GitReference};
1313
use sources::ReplacedSource;
1414
use util::{Config, ToUrl};
1515
use util::config::ConfigValue;
@@ -107,19 +107,25 @@ impl<'cfg> SourceConfigMap<'cfg> {
107107
}
108108
let new_src = new_id.load(self.config)?;
109109
let old_src = id.load(self.config)?;
110-
if new_src.supports_checksums() != old_src.supports_checksums() {
111-
let (supports, no_support) = if new_src.supports_checksums() {
112-
(name, orig_name)
113-
} else {
114-
(orig_name, name)
115-
};
110+
if !new_src.supports_checksums() && old_src.supports_checksums() {
116111
bail!("\
117-
cannot replace `{orig}` with `{name}`, the source `{supports}` supports \
118-
checksums, but `{no_support}` does not
112+
cannot replace `{orig}` with `{name}`, the source `{orig}` supports \
113+
checksums, but `{name}` does not
119114
120115
a lock file compatible with `{orig}` cannot be generated in this situation
121-
", orig = orig_name, name = name, supports = supports, no_support = no_support);
116+
", orig = orig_name, name = name);
117+
}
118+
119+
if old_src.requires_precise() && id.precise().is_none() {
120+
bail!("\
121+
the source {orig} requires a lock file to be present first before it can be
122+
used against vendored source code
123+
124+
remove the source replacement configuration, generate a lock file, and then
125+
restore the source replacement configuration to continue the build
126+
", orig = orig_name);
122127
}
128+
123129
Ok(Box::new(ReplacedSource::new(id, &new_id, new_src)))
124130
}
125131

@@ -153,6 +159,32 @@ a lock file compatible with `{orig}` cannot be generated in this situation
153159
path.push(s);
154160
srcs.push(SourceId::for_directory(&path)?);
155161
}
162+
if let Some(val) = table.get("git") {
163+
let url = url(val, &format!("source.{}.git", name))?;
164+
let try = |s: &str| {
165+
let val = match table.get(s) {
166+
Some(s) => s,
167+
None => return Ok(None),
168+
};
169+
let key = format!("source.{}.{}", name, s);
170+
val.string(&key).map(Some)
171+
};
172+
let reference = match try("branch")? {
173+
Some(b) => GitReference::Branch(b.0.to_string()),
174+
None => {
175+
match try("tag")? {
176+
Some(b) => GitReference::Tag(b.0.to_string()),
177+
None => {
178+
match try("rev")? {
179+
Some(b) => GitReference::Rev(b.0.to_string()),
180+
None => GitReference::Branch("master".to_string()),
181+
}
182+
}
183+
}
184+
}
185+
};
186+
srcs.push(SourceId::for_git(&url, reference)?);
187+
}
156188
if name == "crates-io" && srcs.is_empty() {
157189
srcs.push(SourceId::crates_io(self.config)?);
158190
}

‎src/cargo/sources/directory.rs

+10-3
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ pub struct DirectorySource<'cfg> {
2323

2424
#[derive(Deserialize)]
2525
struct Checksum {
26-
package: String,
26+
package: Option<String>,
2727
files: HashMap<String, String>,
2828
}
2929

@@ -60,6 +60,10 @@ impl<'cfg> Registry for DirectorySource<'cfg> {
6060
fn supports_checksums(&self) -> bool {
6161
true
6262
}
63+
64+
fn requires_precise(&self) -> bool {
65+
true
66+
}
6367
}
6468

6569
impl<'cfg> Source for DirectorySource<'cfg> {
@@ -133,8 +137,11 @@ impl<'cfg> Source for DirectorySource<'cfg> {
133137
})?;
134138

135139
let mut manifest = pkg.manifest().clone();
136-
let summary = manifest.summary().clone();
137-
manifest.set_summary(summary.set_checksum(cksum.package.clone()));
140+
let mut summary = manifest.summary().clone();
141+
if let Some(ref package) = cksum.package {
142+
summary = summary.set_checksum(package.clone());
143+
}
144+
manifest.set_summary(summary);
138145
let pkg = Package::new(manifest, pkg.manifest_path());
139146
self.packages.insert(pkg.package_id().clone(), (pkg, cksum));
140147
}

‎src/cargo/sources/git/source.rs

+8
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,14 @@ impl<'cfg> Registry for GitSource<'cfg> {
130130
.expect("BUG: update() must be called before query()");
131131
src.query(dep, f)
132132
}
133+
134+
fn supports_checksums(&self) -> bool {
135+
false
136+
}
137+
138+
fn requires_precise(&self) -> bool {
139+
true
140+
}
133141
}
134142

135143
impl<'cfg> Source for GitSource<'cfg> {

‎src/cargo/sources/path.rs

+8
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,14 @@ impl<'cfg> Registry for PathSource<'cfg> {
484484
}
485485
Ok(())
486486
}
487+
488+
fn supports_checksums(&self) -> bool {
489+
false
490+
}
491+
492+
fn requires_precise(&self) -> bool {
493+
false
494+
}
487495
}
488496

489497
impl<'cfg> Source for PathSource<'cfg> {

‎src/cargo/sources/registry/mod.rs

+4
Original file line numberDiff line numberDiff line change
@@ -356,6 +356,10 @@ impl<'cfg> Registry for RegistrySource<'cfg> {
356356
fn supports_checksums(&self) -> bool {
357357
true
358358
}
359+
360+
fn requires_precise(&self) -> bool {
361+
false
362+
}
359363
}
360364

361365
impl<'cfg> Source for RegistrySource<'cfg> {

‎src/cargo/sources/replaced.rs

+8
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,14 @@ impl<'cfg> Registry for ReplacedSource<'cfg> {
3333
self.to_replace)
3434
})
3535
}
36+
37+
fn supports_checksums(&self) -> bool {
38+
self.inner.supports_checksums()
39+
}
40+
41+
fn requires_precise(&self) -> bool {
42+
self.inner.requires_precise()
43+
}
3644
}
3745

3846
impl<'cfg> Source for ReplacedSource<'cfg> {

‎src/doc/source-replacement.md

+6
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,12 @@ replace-with = "another-source"
6868
registry = "https://example.com/path/to/index"
6969
local-registry = "path/to/registry"
7070
directory = "path/to/vendor"
71+
72+
# Git sources can optionally specify a branch/tag/rev as well
73+
git = "https://example.com/path/to/repo"
74+
# branch = "master"
75+
# tag = "v1.0.1"
76+
# rev = "313f44e8"
7177
```
7278

7379
The `crates-io` represents the crates.io online registry (default source of

0 commit comments

Comments
 (0)