From 0d4f67cf994989f09686f99aa8e33a855779b149 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 31 Aug 2023 23:04:19 -0700 Subject: [PATCH 1/4] Add registry metadata to published components. This commit adds registry metadata to components published with `cargo component publish`. Closes #110. --- src/commands/publish.rs | 1 + src/lib.rs | 80 +++++++++++++++++++++++----- tests/publish.rs | 112 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 181 insertions(+), 12 deletions(-) diff --git a/src/commands/publish.rs b/src/commands/publish.rs index a1f4baff..a826cdcc 100644 --- a/src/commands/publish.rs +++ b/src/commands/publish.rs @@ -180,6 +180,7 @@ impl PublishCommand { } let options = PublishOptions { + package, registry_url, init: self.init, id, diff --git a/src/lib.rs b/src/lib.rs index 106df889..3fc94f91 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,8 +3,9 @@ #![deny(missing_docs)] use crate::target::install_wasm32_wasi; -use anyhow::{anyhow, bail, Context, Result}; +use anyhow::{bail, Context, Result}; use bindings::BindingsEncoder; +use bytes::Bytes; use cargo_component_core::{ lock::{LockFile, LockFileResolver, LockedPackage, LockedPackageVersion}, registry::create_client, @@ -12,7 +13,6 @@ use cargo_component_core::{ }; use cargo_metadata::{Metadata, MetadataCommand, Package}; use config::{CargoArguments, CargoPackageSpec, Config}; -use futures::TryStreamExt; use lock::{acquire_lock_file_ro, acquire_lock_file_rw}; use metadata::ComponentMetadata; use registry::{PackageDependencyResolution, PackageResolutionMap}; @@ -24,11 +24,10 @@ use std::{ process::Command, time::{Duration, SystemTime}, }; -use tokio::io::BufReader; -use tokio_util::io::ReaderStream; use warg_client::storage::{ContentStorage, PublishEntry, PublishInfo}; use warg_crypto::signing::PrivateKey; use warg_protocol::registry::PackageId; +use wasm_metadata::{Link, LinkType, RegistryMetadata}; use wit_component::ComponentEncoder; mod bindings; @@ -511,6 +510,8 @@ fn create_component(config: &Config, path: &Path, binary: bool) -> Result<()> { /// Represents options for a publish operation. pub struct PublishOptions<'a> { + /// The package to publish. + pub package: &'a Package, /// The registry URL to publish to. pub registry_url: &'a str, /// Whether to initialize the package or not. @@ -527,6 +528,59 @@ pub struct PublishOptions<'a> { pub dry_run: bool, } +fn add_registry_metadata(package: &Package, bytes: &[u8], path: &Path) -> Result> { + let mut metadata = RegistryMetadata::default(); + if !package.authors.is_empty() { + metadata.set_authors(Some(package.authors.clone())); + } + + if !package.categories.is_empty() { + metadata.set_categories(Some(package.categories.clone())); + } + + metadata.set_description(package.description.clone()); + + // TODO: registry metadata should have keywords + // if !package.keywords.is_empty() { + // metadata.set_keywords(Some(package.keywords.clone())); + // } + + metadata.set_license(package.license.clone()); + + let mut links = Vec::new(); + if let Some(docs) = &package.documentation { + links.push(Link { + ty: LinkType::Documentation, + value: docs.clone(), + }); + } + + if let Some(homepage) = &package.homepage { + links.push(Link { + ty: LinkType::Homepage, + value: homepage.clone(), + }); + } + + if let Some(repo) = &package.repository { + links.push(Link { + ty: LinkType::Repository, + value: repo.clone(), + }); + } + + if !links.is_empty() { + metadata.set_links(Some(links)); + } + + metadata.add_to_wasm(bytes).with_context(|| { + format!( + "failed to add registry metadata to component `{path}`", + path = path.display() + ) + }) +} + /// Publish a component for the given workspace and publish options. pub async fn publish(config: &Config, options: &PublishOptions<'_>) -> Result<()> { if options.dry_run { @@ -538,17 +592,19 @@ pub async fn publish(config: &Config, options: &PublishOptions<'_>) -> Result<() let client = create_client(config.warg(), options.registry_url, config.terminal())?; + let bytes = fs::read(options.path).with_context(|| { + format!( + "failed to read component `{path}`", + path = options.path.display() + ) + })?; + + let bytes = add_registry_metadata(options.package, &bytes, options.path)?; + let content = client .content() .store_content( - Box::pin( - ReaderStream::new(BufReader::new( - tokio::fs::File::open(options.path).await.with_context(|| { - format!("failed to open `{path}`", path = options.path.display()) - })?, - )) - .map_err(|e| anyhow!(e)), - ), + Box::pin(futures::stream::once(async { Ok(Bytes::from(bytes)) })), None, ) .await?; diff --git a/tests/publish.rs b/tests/publish.rs index ff47e687..bf48f0c2 100644 --- a/tests/publish.rs +++ b/tests/publish.rs @@ -2,7 +2,12 @@ use crate::support::*; use anyhow::{Context, Result}; use assert_cmd::prelude::*; use predicates::str::contains; +use semver::Version; use std::fs; +use toml_edit::{value, Array}; +use warg_client::Client; +use warg_protocol::registry::PackageId; +use wasm_metadata::LinkType; mod support; @@ -166,3 +171,110 @@ impl Guest for Component { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn it_publishes_with_registry_metadata() -> Result<()> { + let root = create_root()?; + let (_server, config) = spawn_server(&root).await?; + config.write_to_file(&root.join("warg-config.json"))?; + + let authors = ["Jane Doe "]; + let categories = ["wasm"]; + let description = "A test package"; + let license = "Apache-2.0"; + let documentation = "https://example.com/docs"; + let homepage = "https://example.com/home"; + let repository = "https://example.com/repo"; + + let project = Project::with_root(&root, "foo", "")?; + project.update_manifest(|mut doc| { + redirect_bindings_crate(&mut doc); + + let package = &mut doc["package"]; + package["authors"] = value(Array::from_iter(authors)); + package["categories"] = value(Array::from_iter(categories)); + package["description"] = value(description); + package["license"] = value(license); + package["documentation"] = value(documentation); + package["homepage"] = value(homepage); + package["repository"] = value(repository); + Ok(doc) + })?; + + project + .cargo_component("publish --init") + .env("CARGO_COMPONENT_PUBLISH_KEY", test_signing_key()) + .assert() + .stderr(contains("Published package `component:foo` v0.1.0")) + .success(); + + validate_component(&project.release_wasm("foo"))?; + + let client = Client::new_with_config(None, &config)?; + let download = client + .download_exact(&PackageId::new("component:foo")?, &Version::parse("0.1.0")?) + .await?; + + let bytes = fs::read(&download.path).with_context(|| { + format!( + "failed to read downloaded package `{path}`", + path = download.path.display() + ) + })?; + + let metadata = wasm_metadata::RegistryMetadata::from_wasm(&bytes) + .with_context(|| { + format!( + "failed to parse registry metadata from `{path}`", + path = download.path.display() + ) + })? + .expect("missing registry metadata"); + + assert_eq!( + metadata.get_authors().expect("missing authors").as_slice(), + authors + ); + assert_eq!( + metadata + .get_categories() + .expect("missing categories") + .as_slice(), + categories + ); + assert_eq!( + metadata.get_description().expect("missing description"), + description + ); + assert_eq!(metadata.get_license().expect("missing license"), license); + + let links = metadata.get_links().expect("missing links"); + assert_eq!(links.len(), 3); + + assert_eq!( + links + .iter() + .find(|link| link.ty == LinkType::Documentation) + .expect("missing documentation") + .value, + documentation + ); + assert_eq!( + links + .iter() + .find(|link| link.ty == LinkType::Homepage) + .expect("missing homepage") + .value, + homepage + ); + assert_eq!( + links + .iter() + .find(|link| link.ty == LinkType::Repository) + .expect("missing repository") + .value, + repository + ); + + Ok(()) +} From d88e4062df86594d0d8a15f0f9af9df31f19db32 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 31 Aug 2023 23:40:23 -0700 Subject: [PATCH 2/4] Add registry metadata to WIT packages published with `wit`. This commit adds registry metadata to WIT packages published with `wit publish`. The data has been added as optional fields in `wit.toml`. --- crates/wit/src/config.rs | 30 +++++++++ crates/wit/src/lib.rs | 52 +++++++++++++++ crates/wit/tests/publish.rs | 114 +++++++++++++++++++++++++++++++- crates/wit/tests/support/mod.rs | 8 +++ 4 files changed, 202 insertions(+), 2 deletions(-) diff --git a/crates/wit/src/config.rs b/crates/wit/src/config.rs index ad9f742e..6c547550 100644 --- a/crates/wit/src/config.rs +++ b/crates/wit/src/config.rs @@ -62,6 +62,13 @@ impl ConfigBuilder { version: self.version.unwrap_or_else(|| Version::new(0, 1, 0)), dependencies: Default::default(), registries: self.registries, + authors: Default::default(), + categories: Default::default(), + description: None, + license: None, + documentation: None, + homepage: None, + repository: None, } } } @@ -72,9 +79,32 @@ pub struct Config { /// The current package version. pub version: Version, /// The package dependencies. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub dependencies: HashMap, /// The registries to use for sourcing packages. + #[serde(default, skip_serializing_if = "HashMap::is_empty")] pub registries: HashMap, + /// The authors of the package. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub authors: Vec, + /// The categories of the package. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub categories: Vec, + /// The package description. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + /// The package license. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub license: Option, + /// The package documentation URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub documentation: Option, + /// The package homepage URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub homepage: Option, + /// The package repository URL. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub repository: Option, } impl Config { diff --git a/crates/wit/src/lib.rs b/crates/wit/src/lib.rs index 1d1bb16f..ca9e1f10 100644 --- a/crates/wit/src/lib.rs +++ b/crates/wit/src/lib.rs @@ -20,6 +20,7 @@ use std::{ use warg_client::storage::{ContentStorage, PublishEntry, PublishInfo}; use warg_crypto::signing::PrivateKey; use warg_protocol::registry::PackageId; +use wasm_metadata::{Link, LinkType, RegistryMetadata}; use wit_component::DecodedWasm; use wit_parser::{PackageName, Resolve, UnresolvedPackage}; @@ -273,6 +274,56 @@ struct PublishOptions<'a> { dry_run: bool, } +fn add_registry_metadata(config: &Config, bytes: &[u8]) -> Result> { + let mut metadata = RegistryMetadata::default(); + if !config.authors.is_empty() { + metadata.set_authors(Some(config.authors.clone())); + } + + if !config.categories.is_empty() { + metadata.set_categories(Some(config.categories.clone())); + } + + metadata.set_description(config.description.clone()); + + // TODO: registry metadata should have keywords + // if !package.keywords.is_empty() { + // metadata.set_keywords(Some(package.keywords.clone())); + // } + + metadata.set_license(config.license.clone()); + + let mut links = Vec::new(); + if let Some(docs) = &config.documentation { + links.push(Link { + ty: LinkType::Documentation, + value: docs.clone(), + }); + } + + if let Some(homepage) = &config.homepage { + links.push(Link { + ty: LinkType::Homepage, + value: homepage.clone(), + }); + } + + if let Some(repo) = &config.repository { + links.push(Link { + ty: LinkType::Repository, + value: repo.clone(), + }); + } + + if !links.is_empty() { + metadata.set_links(Some(links)); + } + + metadata + .add_to_wasm(bytes) + .context("failed to add registry metadata to component") +} + async fn publish_wit_package(options: PublishOptions<'_>, terminal: &Terminal) -> Result<()> { let (id, bytes) = build_wit_package( options.config, @@ -287,6 +338,7 @@ async fn publish_wit_package(options: PublishOptions<'_>, terminal: &Terminal) - return Ok(()); } + let bytes = add_registry_metadata(options.config, &bytes)?; let id = options.package.unwrap_or(&id); let client = create_client(options.warg_config, options.url, terminal)?; diff --git a/crates/wit/tests/publish.rs b/crates/wit/tests/publish.rs index 677bf8da..3bee43ff 100644 --- a/crates/wit/tests/publish.rs +++ b/crates/wit/tests/publish.rs @@ -1,8 +1,14 @@ +use std::fs; + use crate::support::*; -use anyhow::Result; +use anyhow::{Context, Result}; use assert_cmd::prelude::*; use predicates::str::contains; -use warg_client::FileSystemClient; +use semver::Version; +use toml_edit::{value, Array}; +use warg_client::{Client, FileSystemClient}; +use warg_protocol::registry::PackageId; +use wasm_metadata::LinkType; mod support; @@ -73,3 +79,107 @@ async fn it_does_a_dry_run_publish() -> Result<()> { Ok(()) } + +#[tokio::test(flavor = "multi_thread", worker_threads = 1)] +async fn it_publishes_with_registry_metadata() -> Result<()> { + let root = create_root()?; + let (_server, config) = spawn_server(&root).await?; + config.write_to_file(&root.join("warg-config.json"))?; + + let authors = ["Jane Doe "]; + let categories = ["wasm"]; + let description = "A test package"; + let license = "Apache-2.0"; + let documentation = "https://example.com/docs"; + let homepage = "https://example.com/home"; + let repository = "https://example.com/repo"; + + let project = Project::with_root(&root, "foo", "")?; + project.file("baz.wit", "package baz:qux\n")?; + + project.update_manifest(|mut doc| { + doc["authors"] = value(Array::from_iter(authors)); + doc["categories"] = value(Array::from_iter(categories)); + doc["description"] = value(description); + doc["license"] = value(license); + doc["documentation"] = value(documentation); + doc["homepage"] = value(homepage); + doc["repository"] = value(repository); + Ok(doc) + })?; + + project + .wit("publish --init") + .env("WIT_PUBLISH_KEY", test_signing_key()) + .assert() + .stderr(contains("Published package `baz:qux` v0.1.0")) + .success(); + + let client = Client::new_with_config(None, &config)?; + let download = client + .download_exact(&PackageId::new("baz:qux")?, &Version::parse("0.1.0")?) + .await?; + + let bytes = fs::read(&download.path).with_context(|| { + format!( + "failed to read downloaded package `{path}`", + path = download.path.display() + ) + })?; + + let metadata = wasm_metadata::RegistryMetadata::from_wasm(&bytes) + .with_context(|| { + format!( + "failed to parse registry metadata from `{path}`", + path = download.path.display() + ) + })? + .expect("missing registry metadata"); + + assert_eq!( + metadata.get_authors().expect("missing authors").as_slice(), + authors + ); + assert_eq!( + metadata + .get_categories() + .expect("missing categories") + .as_slice(), + categories + ); + assert_eq!( + metadata.get_description().expect("missing description"), + description + ); + assert_eq!(metadata.get_license().expect("missing license"), license); + + let links = metadata.get_links().expect("missing links"); + assert_eq!(links.len(), 3); + + assert_eq!( + links + .iter() + .find(|link| link.ty == LinkType::Documentation) + .expect("missing documentation") + .value, + documentation + ); + assert_eq!( + links + .iter() + .find(|link| link.ty == LinkType::Homepage) + .expect("missing homepage") + .value, + homepage + ); + assert_eq!( + links + .iter() + .find(|link| link.ty == LinkType::Repository) + .expect("missing repository") + .value, + repository + ); + + Ok(()) +} diff --git a/crates/wit/tests/support/mod.rs b/crates/wit/tests/support/mod.rs index d207a56e..a2590b68 100644 --- a/crates/wit/tests/support/mod.rs +++ b/crates/wit/tests/support/mod.rs @@ -11,6 +11,7 @@ use std::{ }; use tokio::task::JoinHandle; use tokio_util::sync::CancellationToken; +use toml_edit::Document; use warg_crypto::signing::PrivateKey; use warg_server::{policy::content::WasmContentPolicy, Config, Server}; use wasmparser::{Chunk, Encoding, Parser, Payload, Validator, WasmFeatures}; @@ -186,6 +187,13 @@ impl Project { cmd.current_dir(&self.root); cmd } + + pub fn update_manifest(&self, f: impl FnOnce(Document) -> Result) -> Result<()> { + let manifest_path = self.root.join("wit.toml"); + let manifest = fs::read_to_string(&manifest_path)?; + fs::write(manifest_path, f(manifest.parse()?)?.to_string())?; + Ok(()) + } } pub fn validate_component(path: &Path) -> Result<()> { From 9b5e26bda59df46f4d1e507ae4cb9bba3c64f2b5 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Thu, 31 Aug 2023 23:50:57 -0700 Subject: [PATCH 3/4] Remove unnecessary dependency update from `wit add`. This fixes the duplicate updates of package logs when adding a registry dependency via `wit add`. --- crates/wit/src/commands/add.rs | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/crates/wit/src/commands/add.rs b/crates/wit/src/commands/add.rs index 63ccbef4..61f01ced 100644 --- a/crates/wit/src/commands/add.rs +++ b/crates/wit/src/commands/add.rs @@ -1,9 +1,4 @@ -use std::path::PathBuf; - -use crate::{ - config::{Config, CONFIG_FILE_NAME}, - resolve_dependencies, -}; +use crate::config::{Config, CONFIG_FILE_NAME}; use anyhow::{bail, Context, Result}; use cargo_component_core::{ command::CommonOptions, @@ -13,6 +8,7 @@ use cargo_component_core::{ }; use clap::Args; use semver::VersionReq; +use std::path::PathBuf; use warg_protocol::registry::PackageId; async fn resolve_version( @@ -127,16 +123,6 @@ impl AddCommand { .dependencies .insert(id.clone(), Dependency::Package(package)); - // Resolve all dependencies to ensure that the lockfile is up to date. - resolve_dependencies( - &config, - &config_path, - &warg_config, - &terminal, - !self.dry_run, - ) - .await?; - format!( "dependency `{id}` with version `{version}`{dry_run}", dry_run = if self.dry_run { " (dry run)" } else { "" } From 7e3cb0a608367ff5683b1fa011a447d6124fca72 Mon Sep 17 00:00:00 2001 From: Peter Huene Date: Fri, 1 Sep 2023 10:45:59 -0700 Subject: [PATCH 4/4] Update `wit update` tests to ensure lock file creation. --- crates/wit/tests/update.rs | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/crates/wit/tests/update.rs b/crates/wit/tests/update.rs index fab0555b..6f89513d 100644 --- a/crates/wit/tests/update.rs +++ b/crates/wit/tests/update.rs @@ -50,6 +50,12 @@ async fn update_without_changes_is_a_noop() -> Result<()> { .stderr(contains("Added dependency `foo:bar` with version `0.1.0")) .success(); + project + .wit("build") + .assert() + .success() + .stderr(contains("Created package `baz.wasm`")); + project .wit("update") .assert() @@ -82,6 +88,12 @@ async fn test_update_without_compatible_changes_is_a_noop() -> Result<()> { .stderr(contains("Added dependency `foo:bar` with version `0.1.0")) .success(); + project + .wit("build") + .assert() + .success() + .stderr(contains("Created package `baz.wasm`")); + fs::write( root.join("bar/wit.toml"), "version = \"1.0.0\"\n[dependencies]\n[registries]\n", @@ -131,6 +143,12 @@ async fn update_with_compatible_changes() -> Result<()> { .stderr(contains("Added dependency `foo:bar` with version `1.0.0")) .success(); + project + .wit("build") + .assert() + .success() + .stderr(contains("Created package `baz.wasm`")); + fs::write( root.join("bar/wit.toml"), "version = \"1.1.0\"\n[dependencies]\n[registries]\n", @@ -183,6 +201,12 @@ async fn update_with_compatible_changes_is_noop_for_dryrun() -> Result<()> { .stderr(contains("Added dependency `foo:bar` with version `1.0.0")) .success(); + project + .wit("build") + .assert() + .success() + .stderr(contains("Created package `baz.wasm`")); + fs::write( root.join("bar/wit.toml"), "version = \"1.1.0\"\n[dependencies]\n[registries]\n",