build-sys: Rework vendoring for source archive

In preparation for vendoring composefs-rs from git.
Basically before, things work fine when we're just vendoring
from crates.io, but fall over when we add a git dependency.
The Fedora `cargo_prep` macro writes a hardcoded `.cargo/config.toml`
which only has a replacement for `crates.io`, but we need
the generated replacement for git too which is output by
`cargo vendor-filterer` - which previously we were
discarding.

This was surprisingly difficult!

- Capture the output of `vendor-filterer`
- Work around a bug where it puts a broken `directory` path in
  the generated TOML
- Insert that as a new `vendor-config.toml` in our source
- Do use `cargo_prep` to init the RPM config in the spec,
  but re-inject our vendor config appended to that one.

Signed-off-by: Colin Walters <[email protected]>

contrib/packaging/bootc.spec

@@ -64,7 +64,11 @@ Provides: ostree-cli(ostree-container)
 
 %prep
 %autosetup -p1 -a1
-%cargo_prep -v vendor
+# Default -v vendor config doesn't support non-crates.io deps (i.e. git)
+cp .cargo/vendor-config.toml .
+%cargo_prep -N
+cat vendor-config.toml >> .cargo/config.toml
+rm vendor-config.toml
 
 %build
 %if 0%{?fedora} || 0%{?rhel} >= 10
@@ -74,6 +78,8 @@ Provides: ostree-cli(ostree-container)
 %endif
 
 %cargo_vendor_manifest
+# https://pagure.io/fedora-rust/rust-packaging/issue/33
+sed -i -e '/https:\/\//d' cargo-vendor.txt
 %cargo_license_summary
 %{cargo_license} > LICENSE.dependencies
 

xtask/Cargo.toml

@@ -16,6 +16,8 @@ anyhow = { workspace = true }
 camino = { workspace = true }
 chrono = { workspace = true, features = ["std"] }
 fn-error-context = { workspace = true }
+tar = "0.4"
+toml = "0.8"
 tempfile = { workspace = true }
 mandown = "0.1.3"
 xshell = { version = "0.2.6" }

xtask/src/xtask.rs

@@ -3,7 +3,7 @@
 
 use std::fs::File;
 use std::io::{BufRead, BufReader, BufWriter, Write};
-use std::process::{Command, Stdio};
+use std::process::Command;
 
 use anyhow::{anyhow, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
@@ -16,6 +16,13 @@ const TEST_IMAGES: &[&str] = &[
     "quay.io/curl/curl:latest",
     "registry.access.redhat.com/ubi9/podman:latest",
 ];
+const TAR_REPRODUCIBLE_OPTS: &[&str] = &[
+    "--sort=name",
+    "--owner=0",
+    "--group=0",
+    "--numeric-owner",
+    "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime",
+];
 
 fn main() {
     if let Err(e) = try_main() {
@@ -261,6 +268,23 @@ fn git_source_date_epoch(dir: &Utf8Path) -> Result<u64> {
     Ok(r)
 }
 
+/// When using cargo-vendor-filterer --format=tar, the config generated has a bogus source
+/// directory. This edits it to refer to vendor/ as a stable relative reference.
+#[context("Editing vendor config")]
+fn edit_vendor_config(config: &str) -> Result<String> {
+    let mut config: toml::Value = toml::from_str(config)?;
+    let config = config.as_table_mut().unwrap();
+    let source_table = config.get_mut("source").unwrap();
+    let source_table = source_table.as_table_mut().unwrap();
+    let vendored_sources = source_table.get_mut("vendored-sources").unwrap();
+    let vendored_sources = vendored_sources.as_table_mut().unwrap();
+    let previous =
+        vendored_sources.insert("directory".into(), toml::Value::String("vendor".into()));
+    assert!(previous.is_some());
+
+    Ok(config.to_string())
+}
+
 #[context("Packaging")]
 fn impl_package(sh: &Shell) -> Result<Package> {
     let source_date_epoch = git_source_date_epoch(".".into())?;
@@ -269,48 +293,40 @@ fn impl_package(sh: &Shell) -> Result<Package> {
 
     let namev = format!("{NAME}-{v}");
     let p = Utf8Path::new("target").join(format!("{namev}.tar"));
-    let o = File::create(&p)?;
     let prefix = format!("{namev}/");
-    let st = Command::new("git")
-        .args([
-            "archive",
-            "--format=tar",
-            "--prefix",
-            prefix.as_str(),
-            "HEAD",
-        ])
-        .stdout(Stdio::from(o))
-        .status()?;
-    if !st.success() {
-        anyhow::bail!("Failed to run {st:?}");
-    }
-    let st = Command::new("tar")
-        .args([
-            "-r",
-            "-C",
-            "target",
-            "--sort=name",
-            "--owner=0",
-            "--group=0",
-            "--numeric-owner",
-            "--pax-option=exthdr.name=%d/PaxHeaders/%f,delete=atime,delete=ctime",
-        ])
-        .arg(format!("--transform=s,^,{prefix},"))
-        .arg(format!("--mtime=@{source_date_epoch}"))
-        .args(["-f", p.as_str(), "man"])
-        .status()
-        .context("Failed to execute tar")?;
-    if !st.success() {
-        anyhow::bail!("Failed to run {st:?}");
-    }
-    let srcpath: Utf8PathBuf = format!("{p}.zstd").into();
-    cmd!(sh, "zstd --rm -f {p} -o {srcpath}").run()?;
+    cmd!(sh, "git archive --format=tar --prefix={prefix} -o {p} HEAD").run()?;
+    // Generate the vendor directory now, as we want to embed the generated config to use
+    // it in our source.
     let vendorpath = Utf8Path::new("target").join(format!("{namev}-vendor.tar.zstd"));
-    cmd!(
+    let vendor_config = cmd!(
         sh,
         "cargo vendor-filterer --prefix=vendor --format=tar.zstd {vendorpath}"
     )
+    .read()?;
+    let vendor_config = edit_vendor_config(&vendor_config)?;
+    // Append .cargo/vendor-config.toml (a made up filename) into the tar archive.
+    {
+        let tmpdir = tempfile::tempdir_in("target")?;
+        let tmpdir_path = tmpdir.path();
+        let path = tmpdir_path.join("vendor-config.toml");
+        std::fs::write(&path, vendor_config)?;
+        let source_date_epoch = format!("{source_date_epoch}");
+        cmd!(
+            sh,
+            "tar -r -C {tmpdir_path} {TAR_REPRODUCIBLE_OPTS...} --mtime=@{source_date_epoch} --transform=s,^,{prefix}.cargo/, -f {p} vendor-config.toml"
+        )
+        .run()?;
+    }
+    // Append our generated man pages.
+    cmd!(
+        sh,
+        "tar -r -C target {TAR_REPRODUCIBLE_OPTS...} --transform=s,^,{prefix}, -f {p} man"
+    )
     .run()?;
+    // Compress with zstd
+    let srcpath: Utf8PathBuf = format!("{p}.zstd").into();
+    cmd!(sh, "zstd --rm -f {p} -o {srcpath}").run()?;
+
     Ok(Package {
         version: v,
         srcpath,