feat: add direct url repodata building

crates/rattler-bin/Cargo.toml

@@ -34,6 +34,7 @@ rattler_networking = { path="../rattler_networking", version = "0.20.8", default
 rattler_repodata_gateway = { path="../rattler_repodata_gateway", version = "0.20.5", default-features = false, features = ["gateway"] }
 rattler_solve = { path="../rattler_solve", version = "0.24.2", default-features = false, features = ["resolvo", "libsolv_c"] }
 rattler_virtual_packages = { path="../rattler_virtual_packages", version = "0.19.15", default-features = false }
+rattler_cache = { path="../rattler_cache", version = "0.1.0", default-features = false }
 reqwest = { workspace = true }
 reqwest-middleware = { workspace = true }
 tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }

crates/rattler-bin/src/commands/create.rs

@@ -4,6 +4,7 @@ use clap::ValueEnum;
 use indicatif::{ProgressBar, ProgressStyle};
 use itertools::Itertools;
 use rattler::install::{IndicatifReporter, Installer};
+use rattler::package_cache::PackageCache;
 use rattler::{
     default_cache_dir,
     install::{Transaction, TransactionOperation},
@@ -143,7 +144,10 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
 
     // Get the package names from the matchspecs so we can only load the package records that we need.
     let gateway = Gateway::builder()
-        .with_cache_dir(cache_dir.join("repodata"))
+        .with_cache_dir(cache_dir.join(rattler_cache::REPODATA_CACHE_DIR))
+        .with_package_cache(PackageCache::new(
+            cache_dir.join(rattler_cache::PACKAGE_CACHE_DIR),
+        ))
         .with_client(download_client.clone())
         .finish();
 
@@ -204,7 +208,7 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
         "Virtual packages:\n{}\n",
         virtual_packages
             .iter()
-            .format_with("\n", |i, f| f(&format_args!("  - {i}",)),)
+            .format_with("\n", |i, f| f(&format_args!("  - {i}",)))
     );
 
     // Now that we parsed and downloaded all information, construct the packaging problem that we
@@ -283,11 +287,18 @@ pub async fn create(opt: Opt) -> anyhow::Result<()> {
 /// Prints the operations of the transaction to the console.
 fn print_transaction(transaction: &Transaction<PrefixRecord, RepoDataRecord>) {
     let format_record = |r: &RepoDataRecord| {
+        let direct_url_print = if r.clone().channel.is_empty() {
+            r.url.as_str()
+        } else {
+            ""
+        };
+
         format!(
-            "{} {} {}",
+            "{} {} {} {}",
             r.package_record.name.as_normalized(),
             r.package_record.version,
-            r.package_record.build
+            r.package_record.build,
+            direct_url_print,
         )
     };
 

crates/rattler/src/install/installer/mod.rs

@@ -272,7 +272,7 @@ impl Installer {
             PackageCache::new(
                 default_cache_dir()
                     .expect("failed to determine default cache directory")
-                    .join("pkgs"),
+                    .join(rattler_cache::PACKAGE_CACHE_DIR),
             )
         });
 

crates/rattler_cache/src/consts.rs

@@ -0,0 +1,4 @@
+/// The location in the main cache folder where the conda package cache is stored.
+pub const PACKAGE_CACHE_DIR: &str = "pkgs";
+/// The location in the main cache folder where the repodata cache is stored.
+pub const REPODATA_CACHE_DIR: &str = "repodata";

crates/rattler_cache/src/lib.rs

@@ -3,6 +3,10 @@ use std::path::PathBuf;
 pub mod package_cache;
 
 pub mod validation;
+
+mod consts;
+pub use consts::{PACKAGE_CACHE_DIR, REPODATA_CACHE_DIR};
+
 /// Returns the default cache directory used by rattler.
 pub fn default_cache_dir() -> anyhow::Result<PathBuf> {
     Ok(dirs::cache_dir()

crates/rattler_cache/src/package_cache.rs

@@ -62,6 +62,20 @@ pub struct CacheKey {
     sha256: Option<Sha256Hash>,
 }
 
+impl CacheKey {
+    /// Adds a sha256 hash of the archive.
+    pub fn with_sha256(mut self, sha256: Sha256Hash) -> Self {
+        self.sha256 = Some(sha256);
+        self
+    }
+
+    /// Potentially adds a sha256 hash of the archive.
+    pub fn with_opt_sha256(mut self, sha256: Option<Sha256Hash>) -> Self {
+        self.sha256 = sha256;
+        self
+    }
+}
+
 impl CacheKey {
     /// Return the sha256 hash of the package if it is known.
     pub fn sha256(&self) -> Option<Sha256Hash> {
@@ -277,7 +291,7 @@ impl PackageCache {
                     return Err(err);
                 }
 
-                // Determine whether or not to retry based on the retry policy
+                // Determine whether to retry based on the retry policy
                 let execute_after = match retry_policy.should_retry(request_start, current_try) {
                     RetryDecision::Retry { execute_after } => execute_after,
                     RetryDecision::DoNotRetry => return Err(err),

crates/rattler_conda_types/src/lib.rs

@@ -32,7 +32,7 @@ pub use explicit_environment_spec::{
 pub use generic_virtual_package::GenericVirtualPackage;
 pub use match_spec::matcher::{StringMatcher, StringMatcherParseError};
 pub use match_spec::parse::ParseMatchSpecError;
-pub use match_spec::{MatchSpec, NamelessMatchSpec};
+pub use match_spec::{MatchSpec, Matches, NamelessMatchSpec};
 pub use no_arch_type::{NoArchKind, NoArchType};
 pub use package_name::{InvalidPackageNameError, PackageName};
 pub use parse_mode::ParseStrictness;

crates/rattler_conda_types/src/match_spec/mod.rs

@@ -1,4 +1,4 @@
-use crate::{build_spec::BuildNumberSpec, PackageName, PackageRecord, VersionSpec};
+use crate::{build_spec::BuildNumberSpec, PackageName, PackageRecord, RepoDataRecord, VersionSpec};
 use rattler_digest::{serde::SerializableHash, Md5Hash, Sha256Hash};
 use serde::{Deserialize, Deserializer, Serialize};
 use serde_with::{serde_as, skip_serializing_none, DisplayFromStr};
@@ -196,47 +196,6 @@ impl Display for MatchSpec {
 }
 
 impl MatchSpec {
-    /// Match a [`MatchSpec`] against a [`PackageRecord`]
-    pub fn matches(&self, record: &PackageRecord) -> bool {
-        if let Some(name) = self.name.as_ref() {
-            if name != &record.name {
-                return false;
-            }
-        }
-
-        if let Some(spec) = self.version.as_ref() {
-            if !spec.matches(&record.version) {
-                return false;
-            }
-        }
-
-        if let Some(build_string) = self.build.as_ref() {
-            if !build_string.matches(&record.build) {
-                return false;
-            }
-        }
-
-        if let Some(build_number) = self.build_number.as_ref() {
-            if !build_number.matches(&record.build_number) {
-                return false;
-            }
-        }
-
-        if let Some(md5_spec) = self.md5.as_ref() {
-            if Some(md5_spec) != record.md5.as_ref() {
-                return false;
-            }
-        }
-
-        if let Some(sha256_spec) = self.sha256.as_ref() {
-            if Some(sha256_spec) != record.sha256.as_ref() {
-                return false;
-            }
-        }
-
-        true
-    }
-
     /// Decomposes this instance into a [`NamelessMatchSpec`] and a name.
     pub fn into_nameless(self) -> (Option<PackageName>, NamelessMatchSpec) {
         (
@@ -300,37 +259,6 @@ pub struct NamelessMatchSpec {
     pub url: Option<Url>,
 }
 
-impl NamelessMatchSpec {
-    /// Match a [`MatchSpec`] against a [`PackageRecord`]
-    pub fn matches(&self, record: &PackageRecord) -> bool {
-        if let Some(spec) = self.version.as_ref() {
-            if !spec.matches(&record.version) {
-                return false;
-            }
-        }
-
-        if let Some(build_string) = self.build.as_ref() {
-            if !build_string.matches(&record.build) {
-                return false;
-            }
-        }
-
-        if let Some(md5_spec) = self.md5.as_ref() {
-            if Some(md5_spec) != record.md5.as_ref() {
-                return false;
-            }
-        }
-
-        if let Some(sha256_spec) = self.sha256.as_ref() {
-            if Some(sha256_spec) != record.sha256.as_ref() {
-                return false;
-            }
-        }
-
-        true
-    }
-}
-
 impl Display for NamelessMatchSpec {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         match &self.version {
@@ -419,14 +347,136 @@ where
     }
 }
 
+/// A trait that defines the behavior of matching a spec against a record.
+pub trait Matches<T> {
+    /// Match a [`MatchSpec`] against a record.
+    /// Matching it to a record means that the record is valid for the spec.
+    fn matches(&self, other: &T) -> bool;
+}
+
+impl Matches<PackageRecord> for NamelessMatchSpec {
+    /// Match a [`NamelessMatchSpec`] against a [`PackageRecord`]
+    fn matches(&self, other: &PackageRecord) -> bool {
+        if let Some(spec) = self.version.as_ref() {
+            if !spec.matches(&other.version) {
+                return false;
+            }
+        }
+
+        if let Some(build_string) = self.build.as_ref() {
+            if !build_string.matches(&other.build) {
+                return false;
+            }
+        }
+
+        if let Some(build_number) = self.build_number.as_ref() {
+            if !build_number.matches(&other.build_number) {
+                return false;
+            }
+        }
+
+        if let Some(md5_spec) = self.md5.as_ref() {
+            if Some(md5_spec) != other.md5.as_ref() {
+                return false;
+            }
+        }
+
+        if let Some(sha256_spec) = self.sha256.as_ref() {
+            if Some(sha256_spec) != other.sha256.as_ref() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+impl Matches<PackageRecord> for MatchSpec {
+    /// Match a [`MatchSpec`] against a [`PackageRecord`]
+    fn matches(&self, other: &PackageRecord) -> bool {
+        if let Some(name) = self.name.as_ref() {
+            if name != &other.name {
+                return false;
+            }
+        }
+
+        if let Some(spec) = self.version.as_ref() {
+            if !spec.matches(&other.version) {
+                return false;
+            }
+        }
+
+        if let Some(build_string) = self.build.as_ref() {
+            if !build_string.matches(&other.build) {
+                return false;
+            }
+        }
+
+        if let Some(build_number) = self.build_number.as_ref() {
+            if !build_number.matches(&other.build_number) {
+                return false;
+            }
+        }
+
+        if let Some(md5_spec) = self.md5.as_ref() {
+            if Some(md5_spec) != other.md5.as_ref() {
+                return false;
+            }
+        }
+
+        if let Some(sha256_spec) = self.sha256.as_ref() {
+            if Some(sha256_spec) != other.sha256.as_ref() {
+                return false;
+            }
+        }
+
+        true
+    }
+}
+
+impl Matches<RepoDataRecord> for MatchSpec {
+    /// Match a [`MatchSpec`] against a [`RepoDataRecord`]
+    fn matches(&self, other: &RepoDataRecord) -> bool {
+        if let Some(url_spec) = self.url.as_ref() {
+            if url_spec != &other.url {
+                return false;
+            }
+        }
+
+        if !self.matches(&other.package_record) {
+            return false;
+        }
+
+        true
+    }
+}
+
+impl Matches<RepoDataRecord> for NamelessMatchSpec {
+    /// Match a [`NamelessMatchSpec`] against a [`RepoDataRecord`]
+    fn matches(&self, other: &RepoDataRecord) -> bool {
+        if let Some(url_spec) = self.url.as_ref() {
+            if url_spec != &other.url {
+                return false;
+            }
+        }
+
+        if !self.matches(&other.package_record) {
+            return false;
+        }
+
+        true
+    }
+}
+
 #[cfg(test)]
 mod tests {
     use std::str::FromStr;
 
     use rattler_digest::{parse_digest_from_hex, Md5, Sha256};
 
     use crate::{
-        MatchSpec, NamelessMatchSpec, PackageName, PackageRecord, ParseStrictness::*, Version,
+        match_spec::Matches, MatchSpec, NamelessMatchSpec, PackageName, PackageRecord,
+        ParseStrictness::*, RepoDataRecord, Version,
     };
     use insta::assert_snapshot;
     use std::hash::{Hash, Hasher};
@@ -524,6 +574,40 @@ mod tests {
         assert!(!spec.matches(&record));
     }
 
+    #[test]
+    fn test_layered_matches() {
+        let repodata_record = RepoDataRecord {
+            package_record: PackageRecord::new(
+                PackageName::new_unchecked("mamba"),
+                Version::from_str("1.0").unwrap(),
+                String::from(""),
+            ),
+            file_name: String::from("mamba-1.0-py37_0"),
+            url: url::Url::parse("https://mamba.io/mamba-1.0-py37_0.conda").unwrap(),
+            channel: String::from("mamba"),
+        };
+        let package_record = repodata_record.clone().package_record;
+
+        // Test with basic spec
+        let match_spec = MatchSpec::from_str("mamba[version==1.0]", Strict).unwrap();
+        let nameless_spec = match_spec.clone().into_nameless().1;
+
+        assert!(match_spec.matches(&repodata_record));
+        assert!(match_spec.matches(&package_record));
+        assert!(nameless_spec.matches(&repodata_record));
+        assert!(nameless_spec.matches(&package_record));
+
+        // Test with url spec
+        let match_spec =
+            MatchSpec::from_str("https://mamba.io/mamba-1.0-py37_0.conda", Strict).unwrap();
+        let nameless_spec = match_spec.clone().into_nameless().1;
+
+        assert!(match_spec.matches(&repodata_record));
+        assert!(match_spec.matches(&package_record));
+        assert!(nameless_spec.matches(&repodata_record));
+        assert!(nameless_spec.matches(&package_record));
+    }
+
     #[test]
     fn test_serialize_matchspec() {
         let specs = ["mamba 1.0 py37_0",