aur_fetch/
fetch.rs

1use crate::{Callback, CommandFailed, Error};
2
3use std::env::{self, current_dir};
4use std::ffi::OsStr;
5use std::fs::{create_dir_all, File};
6use std::io::{self, Write};
7use std::os::unix::fs::symlink;
8use std::path::{Path, PathBuf};
9use std::process::{Command, Output};
10use std::sync::atomic::{AtomicBool, Ordering};
11
12use crossbeam::channel;
13use url::Url;
14
15static SEEN: &str = "AUR_SEEN";
16
17/// Result type for this crate;
18pub type Result<T> = std::result::Result<T, Error>;
19
20/// Represents a git repository.
21pub struct Repo {
22    /// The url to the git repo.
23    pub url: Url,
24    /// The name of the git repo.
25    pub name: String,
26}
27
28/// Handle to the current configuration.
29///
30/// This handle is used to configure parts of the fetching process. All the features of this crate
31/// must be done through this handle.
32#[derive(Clone, Debug)]
33pub struct Fetch {
34    /// The directory to place AUR packages in.
35    pub clone_dir: PathBuf,
36    /// The directory to place diffs in.
37    pub diff_dir: PathBuf,
38    /// The git command to run.
39    pub git: PathBuf,
40    /// Flags passed to git.
41    pub git_flags: Vec<String>,
42    /// The AUR URL.
43    pub aur_url: Url,
44}
45
46fn command_err(cmd: &Command, stderr: Option<String>) -> Error {
47    Error::CommandFailed(CommandFailed {
48        dir: cmd.get_current_dir().unwrap().to_owned(),
49        command: cmd.get_program().to_owned().into(),
50        args: cmd
51            .get_args()
52            .map(|s| s.to_string_lossy().into_owned())
53            .collect(),
54        stderr,
55    })
56}
57
58impl Fetch {
59    /// Create a new Handle with working defaults.
60    ///
61    /// This Inializes the clone and diff dir to the current dirrectory. If you want to configure
62    /// a cache directory you will need to do that yourself.
63    pub fn new() -> Result<Self> {
64        Ok(Self {
65            clone_dir: env::current_dir()?,
66            diff_dir: env::current_dir()?,
67            git: "git".into(),
68            git_flags: Vec::new(),
69            aur_url: "https://aur.archlinux.org".parse().unwrap(),
70        })
71    }
72
73    /// Create a new handle with a specified cache dir.
74    ///
75    /// clone_dir will be a subdirectory named clone inside of the specified path.
76    /// diff_dir will be a subdirectory named diff inside of the specified path.
77    pub fn with_cache_dir<P: AsRef<Path>>(path: P) -> Self {
78        let path = path.as_ref();
79
80        Self {
81            clone_dir: path.join("clone"),
82            diff_dir: path.join("diff"),
83            git: "git".into(),
84            git_flags: Vec::new(),
85            aur_url: "https://aur.archlinux.org".parse().unwrap(),
86        }
87    }
88
89    /// Create a new handle with a specified cache dir.
90    ///
91    ///Both diffs and cloned packages will be places in the provided dir.
92    pub fn with_combined_cache_dir<P: AsRef<Path>>(path: P) -> Self {
93        let path = path.as_ref();
94
95        Self {
96            clone_dir: path.into(),
97            diff_dir: path.into(),
98            git: "git".into(),
99            git_flags: Vec::new(),
100            aur_url: "https://aur.archlinux.org".parse().unwrap(),
101        }
102    }
103
104    /// Downloads a list of packages to the cache dir.
105    ///
106    /// This downloads packages from the AUR using git. New packages will be cloned, while packages
107    /// that already exist in cache will be fetched. Merging will need to be done in a separate
108    /// step.
109    ///
110    /// Each package is downloaded concurrently which givess a major speedup
111    ///
112    /// Depending on how many packages are being downloaded and connection speed this
113    /// function may take a little while to complete. See [`download_cb`](fn.download_cb.html) if
114    /// you wish track the progress of each download.
115    ///
116    /// This also filters the input list to packages that were already in cache. This filtered list
117    /// can then be passed on to [`merge`](fn.merge.html) as freshly cloned packages will
118    /// not need to be merged.
119    pub fn download<S: AsRef<str> + Send + Sync>(&self, pkgs: &[S]) -> Result<Vec<String>> {
120        self.download_cb(pkgs, |_| ())
121    }
122
123    /// The same as [`download`](fn.download.html) but calls a Callback after each download.
124    ///
125    /// The callback is called each time a package download is completed.
126    pub fn download_cb<S: AsRef<str> + Send + Sync, F: Fn(Callback)>(
127        &self,
128        pkgs: &[S],
129        f: F,
130    ) -> Result<Vec<String>> {
131        let repos = pkgs
132            .iter()
133            .map(|p| {
134                let mut url = self.aur_url.clone();
135                url.set_path(p.as_ref());
136                Repo {
137                    url,
138                    name: p.as_ref().to_string(),
139                }
140            })
141            .collect::<Vec<_>>();
142        self.download_repos_cb(&repos, f)
143    }
144
145    /// The same as [`download`](fn.download.html) but downloads a specified list of repos instead of AUR packages.
146    pub fn download_repos<F: Fn(Callback)>(&self, repos: &[Repo]) -> Result<Vec<String>> {
147        self.download_repos_cb(repos, |_| ())
148    }
149
150    /// The same as [`download_repos`](fn.download_repos.html) but calls a Callback after each download.
151    ///
152    /// The callback is called each time a package download is completed.
153    pub fn download_repos_cb<F: Fn(Callback)>(&self, repos: &[Repo], f: F) -> Result<Vec<String>> {
154        let (pkg_send, pkg_rec) = channel::bounded(0);
155        let (fetched_send, fetched_rec) = channel::bounded(32);
156        let f = &f;
157        let stop = &AtomicBool::new(false);
158        let mut fetched = Vec::with_capacity(repos.len());
159
160        std::thread::scope(|scope| {
161            scope.spawn(move || {
162                for repo in repos {
163                    if pkg_send.send(repo).is_err() {
164                        break;
165                    }
166                }
167            });
168
169            for _ in 0..20.min(repos.len()) {
170                let fetched_send = fetched_send.clone();
171                let pkg_rec = pkg_rec.clone();
172                scope.spawn(move || {
173                    for repo in &pkg_rec {
174                        if stop.load(Ordering::Acquire) {
175                            break;
176                        }
177                        match self.download_pkg(&repo.url, &repo.name) {
178                            Ok((fetched, out)) => {
179                                let _ = fetched_send.send(Ok((repo.name.clone(), fetched, out)));
180                            }
181                            Err(e) => {
182                                stop.store(true, Ordering::Release);
183                                let _ = fetched_send.send(Err(e));
184                                break;
185                            }
186                        }
187                    }
188                });
189            }
190
191            drop(pkg_rec);
192            drop(fetched_send);
193
194            for (n, msg) in fetched_rec.into_iter().enumerate() {
195                let (pkg, was_fetched, out) = msg?;
196                f(Callback {
197                    pkg: &pkg,
198                    n: n + 1,
199                    output: String::from_utf8_lossy(&out).trim(),
200                });
201                if was_fetched {
202                    fetched.push(pkg)
203                }
204            }
205
206            Ok(fetched)
207        })
208    }
209
210    fn download_pkg<S: AsRef<str>>(&self, url: &Url, dir: S) -> Result<(bool, Vec<u8>)> {
211        self.mk_clone_dir()?;
212
213        let dir = dir.as_ref();
214        let is_git_repo = self.is_git_repo(dir);
215
216        let mut command = Command::new(&self.git);
217
218        let fetched = if is_git_repo {
219            command.current_dir(self.clone_dir.join(dir));
220            command.args(["fetch", "-v"]);
221            true
222        } else {
223            command.current_dir(&self.clone_dir);
224            command.args(["clone", "--no-progress", "--", url.as_str(), dir]);
225            false
226        };
227        log_cmd(&command);
228        let output = command
229            .output()
230            .map_err(|e| command_err(&command, Some(e.to_string())))?;
231
232        if !output.status.success() {
233            return Err(command_err(
234                &command,
235                Some(String::from_utf8_lossy(&output.stderr).into_owned()),
236            ));
237        }
238
239        Ok((fetched, output.stderr))
240    }
241
242    /// Filters a list of packages, keep ones that have a diff.
243    ///
244    /// A reoo has a diff if AUR_SEEN is defined and is different to the upstram HEAD.
245    pub fn has_diff<'a, S: AsRef<str>>(&self, pkgs: &'a [S]) -> Result<Vec<&'a str>> {
246        let mut ret = Vec::new();
247
248        for pkg in pkgs {
249            if git_has_diff(
250                &self.git,
251                &self.git_flags,
252                self.clone_dir.join(pkg.as_ref()),
253            )? {
254                ret.push(pkg.as_ref());
255            }
256        }
257
258        Ok(ret)
259    }
260
261    /// Filterrs a list of packages, keeping ones that have not yet been seen.
262    ///
263    /// A repo is seen if AUR_SEEN exists and is equal to the upstram HEAD.
264    pub fn unseen<'a, S: AsRef<str>>(&self, pkgs: &'a [S]) -> Result<Vec<&'a str>> {
265        let mut ret = Vec::new();
266
267        for pkg in pkgs {
268            if git_unseen(
269                &self.git,
270                &self.git_flags,
271                self.clone_dir.join(pkg.as_ref()),
272            )? {
273                ret.push(pkg.as_ref());
274            }
275        }
276
277        Ok(ret)
278    }
279
280    /// Diff a list of packages returning the diffs as strings.
281    ///
282    /// Diffing a package that is already up to date will generate a diff against an empty git tree
283    ///
284    /// Additionally this function gives you the ability to force color. This is useful if you
285    /// intend to print the diffs to stdout.
286    pub fn diff<S: AsRef<str>>(&self, pkgs: &[S], color: bool) -> Result<Vec<String>> {
287        let pkgs = pkgs.iter();
288        let mut ret = Vec::new();
289
290        for pkg in pkgs {
291            let output = git_log(
292                &self.git,
293                &self.git_flags,
294                self.clone_dir.join(pkg.as_ref()),
295                color,
296            )?;
297            let mut s: String = String::from_utf8_lossy(&output.stdout).into();
298            let output = git_diff(
299                &self.git,
300                &self.git_flags,
301                self.clone_dir.join(pkg.as_ref()),
302                color,
303            )?;
304            s.push_str(&String::from_utf8_lossy(&output.stdout));
305            s.push('\n');
306            ret.push(s);
307        }
308
309        Ok(ret)
310    }
311
312    /// Diff a single package.
313    ///
314    /// Relies on `git diff` for printing. This means output will likley be coloured and ran through less.
315    /// Although this is dependent on the user's git config
316    pub fn print_diff<S: AsRef<str>>(&self, pkg: S) -> Result<()> {
317        show_git_diff(
318            &self.git,
319            &self.git_flags,
320            self.clone_dir.join(pkg.as_ref()),
321        )
322    }
323
324    /// Diff a list of packages and save them to diff_dir.
325    ///
326    /// Diffing a package that is already up to date will generate a diff against an empty git tree
327    pub fn save_diffs<S: AsRef<str>>(&self, pkgs: &[S]) -> Result<()> {
328        self.mk_diff_dir()?;
329
330        for pkg in pkgs {
331            let mut path = self.diff_dir.join(pkg.as_ref());
332            path.set_extension("diff");
333
334            let mut file = File::create(path)?;
335
336            file.write_all(
337                &git_log(
338                    &self.git,
339                    &self.git_flags,
340                    self.clone_dir.join(pkg.as_ref()),
341                    false,
342                )?
343                .stdout,
344            )?;
345            file.write_all(b"\n")?;
346            file.write_all(
347                &git_diff(
348                    &self.git,
349                    &self.git_flags,
350                    self.clone_dir.join(pkg.as_ref()),
351                    false,
352                )?
353                .stdout,
354            )?;
355        }
356
357        Ok(())
358    }
359
360    /// Makes a view of newly downloaded files.
361    ///
362    /// This view is a dir containing the packages downloaded/fetched and diffs
363    /// for packages that have diffs.
364    ///
365    /// Files are symlinked from the cache dirs so there is no duplication of files.
366    pub fn make_view<P: AsRef<Path>, S1: AsRef<str>, S2: AsRef<str>>(
367        &self,
368        dir: P,
369        pkgs: &[S1],
370        diffs: &[S2],
371    ) -> Result<()> {
372        let dir = dir.as_ref();
373
374        for pkg in diffs {
375            let pkg = format!("{}.diff", pkg.as_ref());
376            let dest = dir.join(&pkg);
377            let src = self.diff_dir.join(&pkg);
378            if src.is_file() {
379                symlink(src, dest)?;
380            }
381        }
382
383        for pkg in pkgs {
384            let dest = dir.join(pkg.as_ref());
385            let pkgbuild_dest = dir.join(format!("{}.PKGBUILD", pkg.as_ref()));
386            let srcinfo_dest = dir.join(format!("{}.SRCINFO", pkg.as_ref()));
387
388            let src = self.clone_dir.join(pkg.as_ref());
389            if src.is_dir() {
390                symlink(src, dest)?;
391            }
392
393            let src = self.clone_dir.join(pkg.as_ref()).join("PKGBUILD");
394            if src.is_file() {
395                symlink(src, pkgbuild_dest)?;
396            }
397
398            let src = self.clone_dir.join(pkg.as_ref()).join(".SRCINFO");
399            if src.is_file() {
400                symlink(src, srcinfo_dest)?;
401            }
402        }
403
404        Ok(())
405    }
406
407    /// Merge a list of packages with their upstream.
408    pub fn merge<S: AsRef<str>>(&self, pkgs: &[S]) -> Result<()> {
409        self.merge_cb(pkgs, |_| ())
410    }
411
412    /// Merge a list of packages with their upstream, calling callback for each merge.
413    pub fn merge_cb<S: AsRef<str>, F: Fn(Callback)>(&self, pkgs: &[S], cb: F) -> Result<()> {
414        let pkgs = pkgs.iter();
415
416        for (n, pkg) in pkgs.enumerate() {
417            let path = self.clone_dir.join(pkg.as_ref());
418            let output = git_rebase(&self.git, &self.git_flags, path)?;
419            cb(Callback {
420                pkg: pkg.as_ref(),
421                n,
422                output: String::from_utf8_lossy(&output.stdout).trim(),
423            });
424        }
425
426        Ok(())
427    }
428
429    /// Marks a list of repos as seen.
430    ///
431    /// This updates AUR_SEEN to the upstream HEAD
432    pub fn mark_seen<S: AsRef<str>>(&self, pkgs: &[S]) -> Result<()> {
433        for pkg in pkgs {
434            let path = self.clone_dir.join(pkg.as_ref());
435            git_mark_seen(&self.git, &self.git_flags, path)?;
436        }
437
438        Ok(())
439    }
440
441    /// Commits changes to list of packages
442    ///
443    /// This is intended to allow saving changes made by the user after reviewing.
444    pub fn commit<S1: AsRef<str>, S2: AsRef<str>>(&self, pkgs: &[S1], message: S2) -> Result<()> {
445        for pkg in pkgs {
446            let path = self.clone_dir.join(pkg.as_ref());
447            git_commit(&self.git, &self.git_flags, path, message.as_ref())?;
448        }
449
450        Ok(())
451    }
452
453    /// Check if a package is already cloned.
454    pub fn is_git_repo<S: AsRef<str>>(&self, pkg: S) -> bool {
455        self.clone_dir.join(pkg.as_ref()).join(".git").is_dir()
456    }
457
458    fn mk_clone_dir(&self) -> io::Result<()> {
459        create_dir_all(&self.clone_dir)
460    }
461
462    fn mk_diff_dir(&self) -> io::Result<()> {
463        create_dir_all(&self.diff_dir)
464    }
465}
466
467fn color_str(color: bool) -> &'static str {
468    if color {
469        "--color=always"
470    } else {
471        "--color=never"
472    }
473}
474
475fn git_command<S: AsRef<OsStr>, P: AsRef<Path>>(
476    git: S,
477    path: P,
478    flags: &[String],
479    args: &[&str],
480) -> Result<Output> {
481    let mut command = Command::new(git.as_ref());
482    command
483        .current_dir(path.as_ref())
484        .args(flags)
485        .args(args)
486        .env("GIT_TERMINAL_PROMPT", "0");
487
488    log_cmd(&command);
489    let output = command
490        .output()
491        .map_err(|e| command_err(&command, Some(e.to_string())))?;
492
493    if output.status.success() {
494        Ok(output)
495    } else {
496        Err(command_err(
497            &command,
498            Some(String::from_utf8_lossy(&output.stderr).into()),
499        ))
500    }
501}
502
503fn show_git_command<S: AsRef<OsStr>, P: AsRef<Path>>(
504    git: S,
505    path: P,
506    flags: &[String],
507    args: &[&str],
508) -> Result<()> {
509    let mut command = Command::new(git.as_ref());
510    command
511        .current_dir(path.as_ref())
512        .args(flags)
513        .args(args)
514        .env("GIT_TERMINAL_PROMPT", "0");
515
516    log_cmd(&command);
517    let status = command
518        .spawn()
519        .map_err(|e| command_err(&command, Some(e.to_string())))?
520        .wait()
521        .map_err(|e| command_err(&command, Some(e.to_string())))?;
522
523    if status.success() {
524        Ok(())
525    } else {
526        Err(command_err(&command, None))
527    }
528}
529
530fn git_mark_seen<S: AsRef<OsStr>, P: AsRef<Path>>(
531    git: S,
532    flags: &[String],
533    path: P,
534) -> Result<Output> {
535    git_command(&git, &path, flags, &["update-ref", SEEN, "HEAD"])
536}
537
538fn git_rebase<S: AsRef<OsStr>, P: AsRef<Path>>(
539    git: S,
540    flags: &[String],
541    path: P,
542) -> Result<Output> {
543    git_command(&git, &path, flags, &["reset", "--hard", "-q", "HEAD"])?;
544    if git_command(&git, &path, flags, &["symbolic-ref", "-q", "HEAD"]).is_err() {
545        git_command(&git, &path, flags, &["checkout", "master"])?;
546    }
547    git_command(&git, &path, flags, &["rebase", "--stat"])
548}
549
550fn git_unseen<S: AsRef<OsStr>, P: AsRef<Path>>(git: S, flags: &[String], path: P) -> Result<bool> {
551    if git_has_seen(&git, flags, &path)? {
552        let is_unseen = git_command(
553            git,
554            path,
555            flags,
556            &["merge-base", "--is-ancestor", "HEAD@{u}", "AUR_SEEN"],
557        )
558        .is_err();
559        Ok(is_unseen)
560    } else {
561        Ok(true)
562    }
563}
564
565fn git_has_diff<S: AsRef<OsStr>, P: AsRef<Path>>(
566    git: S,
567    flags: &[String],
568    path: P,
569) -> Result<bool> {
570    if git_has_seen(&git, flags, &path)? {
571        let output = git_command(git, path, flags, &["rev-parse", SEEN, "HEAD@{u}"])?;
572
573        let s = String::from_utf8_lossy(&output.stdout);
574        let mut s = s.split('\n');
575
576        let head = s.next().unwrap();
577        let upstream = s.next().unwrap();
578
579        Ok(head != upstream)
580    } else {
581        Ok(false)
582    }
583}
584
585fn git_log<S: AsRef<OsStr>, P: AsRef<Path>>(
586    git: S,
587    flags: &[String],
588    path: P,
589    color: bool,
590) -> Result<Output> {
591    let color = color_str(color);
592    git_command(git, path, flags, &["log", "..HEAD@{u}", color])
593}
594
595fn git_has_seen<S: AsRef<OsStr>, P: AsRef<Path>>(
596    git: S,
597    flags: &[String],
598    path: P,
599) -> Result<bool> {
600    let output = git_command(&git, &path, flags, &["rev-parse", "--verify", SEEN]).is_ok();
601    Ok(output)
602}
603
604fn git_head<S: AsRef<OsStr>, P: AsRef<Path>>(git: S, flags: &[String], path: P) -> Result<String> {
605    let output = git_command(git, path, flags, &["rev-parse", "HEAD"])?;
606    let output = String::from_utf8_lossy(&output.stdout);
607    Ok(output.trim().to_string())
608}
609
610fn git_diff<S: AsRef<OsStr>, P: AsRef<Path>>(
611    git: S,
612    flags: &[String],
613    path: P,
614    color: bool,
615) -> Result<Output> {
616    let color = color_str(color);
617    let head = git_head(&git, flags, &path)?;
618    let output = if git_has_seen(&git, flags, &path)? {
619        git_command(&git, &path, flags, &["reset", "--hard", SEEN])?;
620        git_command(
621            &git,
622            &path,
623            flags,
624            &[
625                "-c",
626                "user.email=aur",
627                "-c",
628                "user.name=aur",
629                "merge",
630                "--no-edit",
631                "--no-ff",
632                "--no-commit",
633            ],
634        )?;
635        Ok(git_command(
636            &git,
637            &path,
638            flags,
639            &[
640                "diff",
641                "--stat",
642                "--patch",
643                "--cached",
644                color,
645                "--",
646                ":!.SRCINFO",
647            ],
648        )?)
649    } else {
650        Ok(git_command(
651            &git,
652            &path,
653            flags,
654            &[
655                "diff",
656                "--stat",
657                "--patch",
658                color,
659                "4b825dc642cb6eb9a060e54bf8d69288fbee4904..HEAD@{u}",
660                "--",
661                ":!.SRCINFO",
662            ],
663        )?)
664    };
665
666    git_command(&git, &path, flags, &["reset", "--hard", &head])?;
667    output
668}
669
670fn show_git_diff<S: AsRef<OsStr>, P: AsRef<Path>>(git: S, flags: &[String], path: P) -> Result<()> {
671    let head = git_head(&git, flags, &path)?;
672    if git_has_seen(&git, flags, &path)? {
673        git_command(&git, &path, flags, &["reset", "--hard", SEEN])?;
674        git_command(
675            &git,
676            &path,
677            flags,
678            &[
679                "-c",
680                "user.email=aur",
681                "-c",
682                "user.name=aur",
683                "merge",
684                "--no-edit",
685                "--no-ff",
686                "--no-commit",
687            ],
688        )?;
689        show_git_command(
690            &git,
691            &path,
692            flags,
693            &["diff", "--stat", "--patch", "--cached", "--", ":!.SRCINFO"],
694        )?;
695    } else {
696        show_git_command(
697            &git,
698            &path,
699            flags,
700            &[
701                "diff",
702                "--stat",
703                "--patch",
704                "4b825dc642cb6eb9a060e54bf8d69288fbee4904..HEAD@{u}",
705                "--",
706                ":!.SRCINFO",
707            ],
708        )?;
709    }
710
711    git_command(&git, &path, flags, &["reset", "--hard", &head])?;
712    Ok(())
713}
714
715fn git_commit<S: AsRef<OsStr>, P: AsRef<Path>>(
716    git: S,
717    flags: &[String],
718    path: P,
719    message: &str,
720) -> Result<()> {
721    let path = path.as_ref();
722    let git = git.as_ref();
723
724    let has_user = git_command(git, path, flags, &["config", "user.name"]).is_ok()
725        && git_command(git, path, flags, &["config", "user.email"]).is_ok();
726
727    if git_command(git, path, flags, &["diff", "--exit-code"]).is_err() {
728        if has_user {
729            git_command(git, path, flags, &["commit", "-am", message])?;
730        } else {
731            git_command(
732                git,
733                path,
734                flags,
735                &[
736                    "-c",
737                    "user.email=aur",
738                    "-c",
739                    "user.name=aur",
740                    "commit",
741                    "-am",
742                    "AUR",
743                ],
744            )?;
745        }
746    }
747
748    Ok(())
749}
750
751fn log_cmd(cmd: &Command) {
752    if log::log_enabled!(log::Level::Debug) {
753        let bin = cmd.get_program().to_string_lossy().to_string();
754        let args = cmd
755            .get_args()
756            .map(|s| s.to_string_lossy().to_string())
757            .collect::<Vec<_>>()
758            .join(" ");
759        let dir = cmd
760            .get_current_dir()
761            .map(|p| p.to_owned())
762            .unwrap_or_else(|| current_dir().unwrap_or_else(|_| "?".into()));
763        let dir = dir.display();
764        log::debug!("running: CWD={dir} {bin} {args}")
765    }
766}
aur_fetch/fetch.rs

aur_fetch/
fetch.rs
