feat: add direct url repodata building

crates/rattler_cache/src/package_cache.rs

@@ -91,6 +91,21 @@ impl From<&PackageRecord> for CacheKey {
     }
 }
 
+impl From<&Url> for CacheKey {
+    fn from(url: &Url) -> Self {
+        if let Some(archive) = ArchiveIdentifier::try_from_url(url) {
+            return archive.into();
+        }
+        // Not a normal archive, so we use the url as the cache key
+        CacheKey {
+            name: url.to_string(),
+            version: "_".to_string(),
+            build_string: "_".to_string(),
+            sha256: None,
+        }
+    }
+}
+
 impl Display for CacheKey {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}-{}-{}", &self.name, &self.version, &self.build_string)

crates/rattler_conda_types/src/match_spec/mod.rs

@@ -327,6 +327,13 @@ impl NamelessMatchSpec {
             }
         }
 
+        // Match on the url, only happens with direct url dependencies
+        if let Some(url_spec) = self.url.as_ref() {
+            if Some(url_spec) != record.direct_url.as_ref() {
+                return false;
+            }
+        }
+
         true
     }
 }

crates/rattler_conda_types/src/match_spec/parse.rs

@@ -20,6 +20,7 @@ use super::{
     matcher::{StringMatcher, StringMatcherParseError},
     MatchSpec,
 };
+use crate::package::ArchiveIdentifier;
 use crate::utils::url::parse_scheme;
 use crate::{
     build_spec::{BuildNumberSpec, ParseBuildNumberSpecError},
@@ -388,8 +389,18 @@ fn matchspec_parser(
     // 2.a Is the spec an url, parse it as an url
     if parse_scheme(input).is_some() {
         let url = Url::parse(input)?;
+
+        let archive = ArchiveIdentifier::try_from_url(&url);
+        let name = archive.and_then(|a| a.try_into().ok());
+
+        // TODO: This should also work without a proper name from the url filename
+        if name.is_none() {
+            return Err(ParseMatchSpecError::MissingPackageName);
+        }
+
         return Ok(MatchSpec {
             url: Some(url),
+            name,
             ..MatchSpec::default()
         });
     }
@@ -398,8 +409,18 @@ fn matchspec_parser(
         let path = Utf8TypedPath::from(input);
         let url = file_url::file_path_to_url(path)
             .map_err(|_error| ParseMatchSpecError::InvalidPackagePathOrUrl)?;
+
+        let archive = ArchiveIdentifier::try_from_url(&url);
+        let name = archive.and_then(|a| a.try_into().ok());
+
+        // TODO: This should also work without a proper name from the url filename
+        if name.is_none() {
+            return Err(ParseMatchSpecError::MissingPackageName);
+        }
+
         return Ok(MatchSpec {
             url: Some(url),
+            name,
             ..MatchSpec::default()
         });
     }
@@ -870,22 +891,16 @@ mod tests {
             spec.url,
             Some(Url::parse("file:/home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2").unwrap())
         );
-
-        let spec = MatchSpec::from_str("C:\\Users\\user\\Downloads\\package", Strict).unwrap();
-        assert_eq!(
-            spec.url,
-            Some(Url::parse("file://C:/Users/user/Downloads/package").unwrap())
-        );
-        let spec = MatchSpec::from_str("/home/user/Downloads/package", Strict).unwrap();
-
-        assert_eq!(
-            spec.url,
-            Some(Url::parse("file:/home/user/Downloads/package").unwrap())
-        );
     }
 
     #[test]
     fn test_non_happy_url_parsing() {
+        let spec = MatchSpec::from_str("C:\\Users\\user\\Downloads\\package", Strict).unwrap_err();
+        assert_matches!(spec, ParseMatchSpecError::MissingPackageName);
+
+        let spec = MatchSpec::from_str("/home/user/Downloads/package", Strict).unwrap_err();
+        assert_matches!(spec, ParseMatchSpecError::MissingPackageName);
+
         let err = MatchSpec::from_str("http://username@", Strict).expect_err("Invalid url");
         assert_eq!(err.to_string(), "invalid package spec url");
 

crates/rattler_conda_types/src/match_spec/snapshots/rattler_conda_types__match_spec__parse__tests__parsed matchspecs.snap

@@ -4,8 +4,10 @@ assertion_line: 794
 expression: evaluated
 ---
 /home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2:
+  name: foo
   url: "file:///home/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2"
 "C:\\Users\\user\\conda-bld\\linux-64\\foo-1.0-py27_0.tar.bz2":
+  name: foo
   url: "file:///C:/Users/user/conda-bld/linux-64/foo-1.0-py27_0.tar.bz2"
 blas *.* mkl:
   name: blas
@@ -35,8 +37,10 @@ foo==1.0=py27_0:
   version: "==1.0"
   build: py27_0
 "https://conda.anaconda.org/conda-forge/linux-64/py-rattler-0.6.1-py39h8169da8_0.conda":
+  name: py-rattler
   url: "https://conda.anaconda.org/conda-forge/linux-64/py-rattler-0.6.1-py39h8169da8_0.conda"
 "https://repo.prefix.dev/ruben-arts/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2":
+  name: boost-cpp
   url: "https://repo.prefix.dev/ruben-arts/linux-64/boost-cpp-1.78.0-h75c5d50_1.tar.bz2"
 python 3.8.* *_cpython:
   name: python

crates/rattler_conda_types/src/package_name.rs

@@ -1,3 +1,4 @@
+use crate::package::ArchiveIdentifier;
 use crate::utils::serde::DeserializeFromStrUnchecked;
 use serde::{Deserialize, Deserializer, Serialize, Serializer};
 use serde_with::{DeserializeAs, DeserializeFromStr};
@@ -16,7 +17,7 @@ use thiserror::Error;
 /// This struct explicitly does not implement [`std::fmt::Display`] because its ambiguous if that
 /// would display the source or the normalized version. Simply call `as_source` or `as_normalized`
 /// to make the distinction.
-#[derive(Debug, Clone, Eq, DeserializeFromStr)]
+#[derive(Debug, Default, Clone, Eq, DeserializeFromStr)]
 pub struct PackageName {
     normalized: Option<String>,
     source: String,
@@ -62,6 +63,14 @@ impl TryFrom<&String> for PackageName {
     }
 }
 
+impl TryFrom<ArchiveIdentifier> for PackageName {
+    type Error = InvalidPackageNameError;
+
+    fn try_from(value: ArchiveIdentifier) -> Result<Self, Self::Error> {
+        value.name.try_into()
+    }
+}
+
 impl TryFrom<String> for PackageName {
     type Error = InvalidPackageNameError;
 

crates/rattler_conda_types/src/repo_data/mod.rs

@@ -80,7 +80,7 @@ pub struct ChannelInfo {
 #[serde_as]
 #[skip_serializing_none]
 #[sorted]
-#[derive(Debug, Deserialize, Serialize, Eq, PartialEq, Clone, Hash)]
+#[derive(Debug, Default, Deserialize, Serialize, Eq, PartialEq, Clone, Hash)]
 pub struct PackageRecord {
     /// Optionally the architecture the package supports
     pub arch: Option<String>,
@@ -103,6 +103,10 @@ pub struct PackageRecord {
     #[serde(default)]
     pub depends: Vec<String>,
 
+    /// The package's url which isn't part of an real pacakge record but injected by the direct url query.
+    #[serde(skip)]
+    pub direct_url: Option<Url>,
+
     /// Features are a deprecated way to specify different feature sets for the
     /// conda solver. This is not supported anymore and should not be used.
     /// Instead, `mutex` packages should be used to specify
@@ -187,7 +191,9 @@ pub struct PackageRecord {
 
 impl Display for PackageRecord {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        if self.build.is_empty() {
+        if let Some(url) = &self.direct_url {
+            write!(f, "{url}")
+        } else if self.build.is_empty() {
             write!(f, "{} {}", self.name.as_normalized(), self.version,)
         } else {
             write!(
@@ -311,6 +317,7 @@ impl PackageRecord {
             version: version.into(),
             purls: None,
             run_exports: None,
+            direct_url: None,
         }
     }
 
@@ -429,8 +436,15 @@ impl PackageRecord {
             version: index.version,
             purls: None,
             run_exports: None,
+            direct_url: None,
         })
     }
+
+    /// Add url to the [`PackageRecord`]
+    pub fn with_package_url(mut self, url: Url) -> Self {
+        self.direct_url = Some(url);
+        self
+    }
 }
 
 fn sort_map_alphabetically<T: Serialize, S: serde::Serializer>(

crates/rattler_conda_types/src/version/mod.rs

@@ -133,7 +133,7 @@ pub use with_source::VersionWithSource;
 /// this problem by appending an underscore to plain version numbers:
 ///
 /// 1.0.1_ < 1.0.1a =>  True   # ensure correct ordering for openssl
-#[derive(Clone, Eq)]
+#[derive(Clone, Eq, Default)]
 pub struct Version {
     /// Individual components of the version.
     ///

crates/rattler_conda_types/src/version/with_source.rs

@@ -22,7 +22,7 @@ use std::{
 ///
 /// It is also possible to convert directly from a [`Version`] but the [`Display`] implementation
 /// is then used to generate the string representation.
-#[derive(Debug, Clone)]
+#[derive(Debug, Default, Clone)]
 pub struct VersionWithSource {
     version: Version,
     source: Option<Box<str>>,

crates/rattler_index/src/lib.rs

@@ -49,6 +49,7 @@ fn package_record_from_index_json<T: Read>(
         legacy_bz2_size: None,
         purls: None,
         run_exports: None,
+        direct_url: None,
     };
 
     Ok(package_record)

crates/rattler_lock/src/parse/v3.rs

@@ -173,6 +173,7 @@ pub fn parse_v3_or_lower(
                             version: value.version,
                             purls: value.purls.is_empty().not().then_some(value.purls),
                             run_exports: None,
+                            direct_url: None,
                         },
                         url: value.url,
                         file_name: None,

crates/rattler_lock/src/utils/serde/raw_conda_package_data.rs

@@ -125,6 +125,7 @@ impl<'a> From<RawCondaPackageData<'a>> for CondaPackageData {
                 track_features: value.track_features.into_owned(),
                 version: value.version.into_owned(),
                 run_exports: None,
+                direct_url: None,
             },
             url: value.url.into_owned(),
             file_name: value.file_name.into_owned(),

crates/rattler_repodata_gateway/Cargo.toml

@@ -54,6 +54,8 @@ tracing = { workspace = true }
 url = { workspace = true, features = ["serde"] }
 zstd = { workspace = true }
 percent-encoding = { workspace = true }
+rattler_cache = { version = "0.1.0", path = "../rattler_cache" }
+rattler_package_streaming = { version = "0.21.2", path = "../rattler_package_streaming", default-features = false, features = ["reqwest"] }
 
 [target.'cfg(unix)'.dependencies]
 libc = { workspace = true }
@@ -72,6 +74,7 @@ tower-http = { workspace = true, features = ["fs", "compression-gzip", "trace"]
 tracing-test = { workspace = true }
 rattler_conda_types = { path = "../rattler_conda_types", version = "0.25.2", default-features = false }
 fslock = { workspace = true }
+tools = { path="../tools" }
 
 [features]
 default = ['native-tls']