cargo_edit_9/
fetch.rs

1use std::collections::BTreeMap;
2use std::env;
3use std::io::Write;
4use std::path::Path;
5use std::time::Duration;
6
7use termcolor::{Color, ColorSpec, StandardStream, WriteColor};
8use url::Url;
9
10use super::errors::*;
11use super::registry::registry_url;
12use super::VersionExt;
13use super::{Dependency, LocalManifest, Manifest};
14use regex::Regex;
15
16/// Query latest version from a registry index
17///
18/// The registry argument must be specified for crates
19/// from alternative registries.
20///
21/// The latest version will be returned as a `Dependency`. This will fail, when
22///
23/// - there is no Internet connection and offline is false.
24/// - summaries in registry index with an incorrect format.
25/// - a crate with the given name does not exist on the registry.
26pub fn get_latest_dependency(
27    crate_name: &str,
28    flag_allow_prerelease: bool,
29    manifest_path: &Path,
30    registry: Option<&Url>,
31) -> CargoResult<Dependency> {
32    if env::var("CARGO_IS_TEST").is_ok() {
33        // We are in a simulated reality. Nothing is real here.
34        // FIXME: Use actual test handling code.
35        let new_version = if flag_allow_prerelease {
36            format!("99999.0.0-alpha.1+{}", crate_name)
37        } else {
38            match crate_name {
39                "test_breaking" => "0.2.0".to_string(),
40                "test_nonbreaking" => "0.1.1".to_string(),
41                other => format!("99999.0.0+{}", other),
42            }
43        };
44
45        let features = if crate_name == "your-face" {
46            [
47                ("nose".to_string(), vec![]),
48                ("mouth".to_string(), vec![]),
49                ("eyes".to_string(), vec![]),
50                ("ears".to_string(), vec![]),
51            ]
52            .into_iter()
53            .collect::<BTreeMap<_, _>>()
54        } else {
55            BTreeMap::default()
56        };
57
58        return Ok(Dependency::new(crate_name)
59            .set_version(&new_version)
60            .set_available_features(features));
61    }
62
63    if crate_name.is_empty() {
64        anyhow::bail!("Found empty crate name");
65    }
66
67    let registry = match registry {
68        Some(url) => url.clone(),
69        None => registry_url(manifest_path, None)?,
70    };
71
72    let crate_versions = fuzzy_query_registry_index(crate_name, &registry)?;
73
74    let dep = read_latest_version(&crate_versions, flag_allow_prerelease)?;
75
76    if dep.name != crate_name {
77        eprintln!("WARN: Added `{}` instead of `{}`", dep.name, crate_name);
78    }
79
80    Ok(dep)
81}
82
83#[derive(Debug)]
84struct CrateVersion {
85    name: String,
86    version: semver::Version,
87    yanked: bool,
88    available_features: BTreeMap<String, Vec<String>>,
89}
90
91/// Fuzzy query crate from registry index
92fn fuzzy_query_registry_index(
93    crate_name: impl Into<String>,
94    registry: &Url,
95) -> CargoResult<Vec<CrateVersion>> {
96    let index = crates_index::Index::from_url(registry.as_str())?;
97
98    let crate_name = crate_name.into();
99    let mut names = gen_fuzzy_crate_names(crate_name.clone())?;
100    if let Some(index) = names.iter().position(|x| *x == crate_name) {
101        // ref: https://github.com/killercup/cargo-edit/pull/317#discussion_r307365704
102        names.swap(index, 0);
103    }
104
105    for the_name in names {
106        let crate_ = match index.crate_(&the_name) {
107            Some(crate_) => crate_,
108            None => continue,
109        };
110        return crate_
111            .versions()
112            .iter()
113            .map(|v| {
114                Ok(CrateVersion {
115                    name: v.name().to_owned(),
116                    version: v.version().parse()?,
117                    yanked: v.is_yanked(),
118                    available_features: registry_features(v),
119                })
120            })
121            .collect();
122    }
123    Err(no_crate_err(crate_name))
124}
125
126/// Generate all similar crate names
127///
128/// Examples:
129///
130/// | input | output |
131/// | ----- | ------ |
132/// | cargo | cargo  |
133/// | cargo-edit | cargo-edit, cargo_edit |
134/// | parking_lot_core | parking_lot_core, parking_lot-core, parking-lot_core, parking-lot-core |
135fn gen_fuzzy_crate_names(crate_name: String) -> CargoResult<Vec<String>> {
136    const PATTERN: [u8; 2] = [b'-', b'_'];
137
138    let wildcard_indexs = crate_name
139        .bytes()
140        .enumerate()
141        .filter(|(_, item)| PATTERN.contains(item))
142        .map(|(index, _)| index)
143        .take(10)
144        .collect::<Vec<usize>>();
145    if wildcard_indexs.is_empty() {
146        return Ok(vec![crate_name]);
147    }
148
149    let mut result = vec![];
150    let mut bytes = crate_name.into_bytes();
151    for mask in 0..2u128.pow(wildcard_indexs.len() as u32) {
152        for (mask_index, wildcard_index) in wildcard_indexs.iter().enumerate() {
153            let mask_value = (mask >> mask_index) & 1 == 1;
154            if mask_value {
155                bytes[*wildcard_index] = b'-';
156            } else {
157                bytes[*wildcard_index] = b'_';
158            }
159        }
160        result.push(String::from_utf8(bytes.clone()).unwrap());
161    }
162    Ok(result)
163}
164
165// Checks whether a version object is a stable release
166fn version_is_stable(version: &CrateVersion) -> bool {
167    !version.version.is_prerelease()
168}
169
170/// Read latest version from Versions structure
171fn read_latest_version(
172    versions: &[CrateVersion],
173    flag_allow_prerelease: bool,
174) -> CargoResult<Dependency> {
175    let latest = versions
176        .iter()
177        .filter(|&v| flag_allow_prerelease || version_is_stable(v))
178        .filter(|&v| !v.yanked)
179        .max_by_key(|&v| v.version.clone())
180        .ok_or_else(|| {
181            anyhow::format_err!(
182                "No available versions exist. Either all were yanked \
183                         or only prerelease versions exist. Trying with the \
184                         --allow-prerelease flag might solve the issue."
185            )
186        })?;
187
188    let name = &latest.name;
189    let version = latest.version.to_string();
190    Ok(Dependency::new(name)
191        .set_version(&version)
192        .set_available_features(latest.available_features.clone()))
193}
194
195/// Get crate features from registry
196pub fn get_features_from_registry(
197    crate_name: &str,
198    version: &str,
199    registry: &Url,
200) -> CargoResult<BTreeMap<String, Vec<String>>> {
201    if env::var("CARGO_IS_TEST").is_ok() {
202        let features = if crate_name == "your-face" {
203            [
204                ("nose".to_string(), vec![]),
205                ("mouth".to_string(), vec![]),
206                ("eyes".to_string(), vec![]),
207                ("ears".to_string(), vec![]),
208            ]
209            .into_iter()
210            .collect::<BTreeMap<_, _>>()
211        } else {
212            BTreeMap::default()
213        };
214        return Ok(features);
215    }
216
217    let index = crates_index::Index::from_url(registry.as_str())?;
218    let version =
219        semver::VersionReq::parse(version).map_err(|_| parse_version_err(version, crate_name))?;
220
221    let crate_ = index
222        .crate_(crate_name)
223        .ok_or_else(|| no_crate_err(crate_name))?;
224    for crate_instance in crate_.versions().iter().rev() {
225        let instance_version = match semver::Version::parse(crate_instance.version()) {
226            Ok(version) => version,
227            Err(_) => continue,
228        };
229        if version.matches(&instance_version) {
230            return Ok(registry_features(crate_instance));
231        }
232    }
233    Ok(registry_features(crate_.highest_version()))
234}
235
236fn registry_features(v: &crates_index::Version) -> BTreeMap<String, Vec<String>> {
237    let mut features: BTreeMap<_, _> = v
238        .features()
239        .iter()
240        .map(|(k, v)| (k.clone(), v.clone()))
241        .collect();
242    features.extend(
243        v.dependencies()
244            .iter()
245            .filter(|d| d.is_optional())
246            .map(|d| (d.crate_name().to_owned(), vec![])),
247    );
248    features
249}
250
251/// update registry index for given project
252pub fn update_registry_index(registry: &Url, quiet: bool) -> CargoResult<()> {
253    let colorchoice = super::colorize_stderr();
254    let mut output = StandardStream::stderr(colorchoice);
255
256    let mut index = crates_index::Index::from_url(registry.as_str())?;
257    if !quiet {
258        output.set_color(ColorSpec::new().set_fg(Some(Color::Green)).set_bold(true))?;
259        write!(output, "{:>12}", "Updating")?;
260        output.reset()?;
261        writeln!(output, " '{}' index", registry)?;
262    }
263
264    while need_retry(index.update())? {
265        registry_blocked_message(&mut output)?;
266        std::thread::sleep(REGISTRY_BACKOFF);
267    }
268
269    Ok(())
270}
271
272/// Time between retries for retrieving the registry.
273const REGISTRY_BACKOFF: Duration = Duration::from_secs(1);
274
275/// Check if we need to retry retrieving the Index.
276fn need_retry(res: Result<(), crates_index::Error>) -> CargoResult<bool> {
277    match res {
278        Ok(()) => Ok(false),
279        Err(crates_index::Error::Git(err)) => {
280            if err.class() == git2::ErrorClass::Index && err.code() == git2::ErrorCode::Locked {
281                Ok(true)
282            } else {
283                Err(crates_index::Error::Git(err).into())
284            }
285        }
286        Err(err) => Err(err.into()),
287    }
288}
289
290/// Report to user that the Registry is locked
291fn registry_blocked_message(output: &mut StandardStream) -> CargoResult<()> {
292    output.set_color(ColorSpec::new().set_fg(Some(Color::Green)).set_bold(true))?;
293    write!(output, "{:>12}", "Blocking")?;
294    output.reset()?;
295    writeln!(output, " waiting for lock on registry index")?;
296    Ok(())
297}
298
299/// Load Cargo.toml in a local path
300///
301/// This will fail, when Cargo.toml is not present in the root of the path.
302pub fn get_manifest_from_path(path: &Path) -> CargoResult<LocalManifest> {
303    let cargo_file = path.join("Cargo.toml");
304    LocalManifest::try_new(&cargo_file).with_context(|| "Unable to open local Cargo.toml")
305}
306
307/// Load Cargo.toml from  github repo Cargo.toml
308///
309/// This will fail when:
310/// - there is no Internet connection,
311/// - Cargo.toml is not present in the root of the master branch,
312/// - the response from the server is an error or in an incorrect format.
313pub fn get_manifest_from_url(url: &str) -> CargoResult<Option<Manifest>> {
314    let manifest = if is_github_url(url) {
315        Some(get_manifest_from_github(url)?)
316    } else if is_gitlab_url(url) {
317        Some(get_manifest_from_gitlab(url)?)
318    } else {
319        None
320    };
321    Ok(manifest)
322}
323
324fn is_github_url(url: &str) -> bool {
325    url.contains("https://github.com")
326}
327
328fn is_gitlab_url(url: &str) -> bool {
329    url.contains("https://gitlab.com")
330}
331
332fn get_manifest_from_github(repo: &str) -> CargoResult<Manifest> {
333    let re =
334        Regex::new(r"^https://github.com/([-_0-9a-zA-Z]+)/([-_0-9a-zA-Z]+)(/|.git)?$").unwrap();
335    get_manifest_from_repository(repo, &re, |user, repo| {
336        format!(
337            "https://raw.githubusercontent.com/{user}/{repo}/master/Cargo.toml",
338            user = user,
339            repo = repo
340        )
341    })
342}
343
344fn get_manifest_from_gitlab(repo: &str) -> CargoResult<Manifest> {
345    let re =
346        Regex::new(r"^https://gitlab.com/([-_0-9a-zA-Z]+)/([-_0-9a-zA-Z]+)(/|.git)?$").unwrap();
347    get_manifest_from_repository(repo, &re, |user, repo| {
348        format!(
349            "https://gitlab.com/{user}/{repo}/raw/master/Cargo.toml",
350            user = user,
351            repo = repo
352        )
353    })
354}
355
356fn get_manifest_from_repository<T>(
357    repo: &str,
358    matcher: &Regex,
359    url_template: T,
360) -> CargoResult<Manifest>
361where
362    T: Fn(&str, &str) -> String,
363{
364    matcher
365        .captures(repo)
366        .ok_or_else(|| anyhow::format_err!("Unable to parse git repo URL"))
367        .and_then(|cap| match (cap.get(1), cap.get(2)) {
368            (Some(user), Some(repo)) => {
369                let url = url_template(user.as_str(), repo.as_str());
370                get_cargo_toml_from_git_url(&url)
371                    .and_then(|m| m.parse().with_context(parse_manifest_err))
372            }
373            _ => Err(anyhow::format_err!("Git repo url seems incomplete")),
374        })
375}
376
377fn get_cargo_toml_from_git_url(url: &str) -> CargoResult<String> {
378    let mut agent = ureq::AgentBuilder::new().timeout(get_default_timeout());
379    #[cfg(not(any(
380        target_arch = "x86_64",
381        target_arch = "arm",
382        target_arch = "x86",
383        target_arch = "aarch64"
384    )))]
385    {
386        use std::sync::Arc;
387
388        let tls_connector = Arc::new(native_tls::TlsConnector::new()?);
389        agent = agent.tls_connector(tls_connector.clone());
390    }
391    if let Some(proxy) = env_proxy::for_url_str(url)
392        .to_url()
393        .and_then(|url| ureq::Proxy::new(url).ok())
394    {
395        agent = agent.proxy(proxy);
396    }
397    let req = agent.build().get(url);
398    let res = req.call();
399    match res {
400        Ok(res) => res
401            .into_string()
402            .with_context(|| "Git response not a valid `String`"),
403        Err(err) => Err(anyhow::format_err!(
404            "HTTP request `{}` failed: {}",
405            url,
406            err
407        )),
408    }
409}
410
411const fn get_default_timeout() -> Duration {
412    Duration::from_secs(10)
413}
414
415#[test]
416fn test_gen_fuzzy_crate_names() {
417    fn test_helper(input: &str, expect: &[&str]) {
418        let mut actual = gen_fuzzy_crate_names(input.to_string()).unwrap();
419        actual.sort();
420
421        let mut expect = expect.iter().map(|x| x.to_string()).collect::<Vec<_>>();
422        expect.sort();
423
424        assert_eq!(actual, expect);
425    }
426
427    test_helper("", &[""]);
428    test_helper("-", &["_", "-"]);
429    test_helper("DCjanus", &["DCjanus"]);
430    test_helper("DC-janus", &["DC-janus", "DC_janus"]);
431    test_helper(
432        "DC-_janus",
433        &["DC__janus", "DC_-janus", "DC-_janus", "DC--janus"],
434    );
435}
436
437#[test]
438fn get_latest_stable_version() {
439    let versions = vec![
440        CrateVersion {
441            name: "foo".into(),
442            version: "0.6.0-alpha".parse().unwrap(),
443            yanked: false,
444            available_features: BTreeMap::new(),
445        },
446        CrateVersion {
447            name: "foo".into(),
448            version: "0.5.0".parse().unwrap(),
449            yanked: false,
450            available_features: BTreeMap::new(),
451        },
452    ];
453    assert_eq!(
454        read_latest_version(&versions, false)
455            .unwrap()
456            .version()
457            .unwrap(),
458        "0.5.0"
459    );
460}
461
462#[test]
463fn get_latest_unstable_or_stable_version() {
464    let versions = vec![
465        CrateVersion {
466            name: "foo".into(),
467            version: "0.6.0-alpha".parse().unwrap(),
468            yanked: false,
469            available_features: BTreeMap::new(),
470        },
471        CrateVersion {
472            name: "foo".into(),
473            version: "0.5.0".parse().unwrap(),
474            yanked: false,
475            available_features: BTreeMap::new(),
476        },
477    ];
478    assert_eq!(
479        read_latest_version(&versions, true)
480            .unwrap()
481            .version()
482            .unwrap(),
483        "0.6.0-alpha"
484    );
485}
486
487#[test]
488fn get_latest_version_with_yanked() {
489    let versions = vec![
490        CrateVersion {
491            name: "treexml".into(),
492            version: "0.3.1".parse().unwrap(),
493            yanked: true,
494            available_features: BTreeMap::new(),
495        },
496        CrateVersion {
497            name: "true".into(),
498            version: "0.3.0".parse().unwrap(),
499            yanked: false,
500            available_features: BTreeMap::new(),
501        },
502    ];
503    assert_eq!(
504        read_latest_version(&versions, false)
505            .unwrap()
506            .version()
507            .unwrap(),
508        "0.3.0"
509    );
510}
511
512#[test]
513fn get_no_latest_version_from_json_when_all_are_yanked() {
514    let versions = vec![
515        CrateVersion {
516            name: "treexml".into(),
517            version: "0.3.1".parse().unwrap(),
518            yanked: true,
519            available_features: BTreeMap::new(),
520        },
521        CrateVersion {
522            name: "true".into(),
523            version: "0.3.0".parse().unwrap(),
524            yanked: true,
525            available_features: BTreeMap::new(),
526        },
527    ];
528    assert!(read_latest_version(&versions, false).is_err());
529}
cargo_edit_9/fetch.rs

cargo_edit_9/
fetch.rs
