diff --git a/Cargo.lock b/Cargo.lock index 44ef3f4b38..605a9cad82 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1830,7 +1830,6 @@ dependencies = [ "anyhow", "async-trait", "derive_builder 0.12.0", - "hyper 0.14.25", "hyper-tls", "nix 0.26.2", "regex", diff --git a/crates/e2e-testing/Cargo.toml b/crates/e2e-testing/Cargo.toml index 7fbd464ec6..de23d7f786 100644 --- a/crates/e2e-testing/Cargo.toml +++ b/crates/e2e-testing/Cargo.toml @@ -10,12 +10,11 @@ doctest = false [dependencies] anyhow = "1.0" async-trait = "0.1" -tokio = { version = "1.23", features = [ "full" ] } -hyper = { version = "0.14", features = [ "full" ] } +tokio = { version = "1.23", features = ["full"] } regex = "1.5.5" reqwest = { version = "0.11", features = ["blocking"] } nix = "0.26.1" url = "2.2.2" derive_builder = "0.12.0" hyper-tls = "0.5.0" -tempfile = "3.3.0" \ No newline at end of file +tempfile = "3.3.0" diff --git a/crates/e2e-testing/src/http_asserts.rs b/crates/e2e-testing/src/http_asserts.rs index 314427e156..5e5133a079 100644 --- a/crates/e2e-testing/src/http_asserts.rs +++ b/crates/e2e-testing/src/http_asserts.rs @@ -1,16 +1,14 @@ use crate::ensure_eq; use anyhow::Result; -use hyper::client::HttpConnector; -use hyper::{body, Body, Client, Method, Request, Response}; -use hyper_tls::HttpsConnector; +use reqwest::{Method, Request, Response}; use std::str; pub async fn assert_status(url: &str, expected: u16) -> Result<()> { let resp = make_request(Method::GET, url, "").await?; let status = resp.status(); - let response = body::to_bytes(resp.into_body()).await.unwrap().to_vec(); - let actual_body = str::from_utf8(&response).unwrap().to_string(); + let body = resp.bytes().await?; + let actual_body = str::from_utf8(&body).unwrap().to_string(); ensure_eq!(status, expected, "{}", actual_body); @@ -29,8 +27,8 @@ pub async fn assert_http_response( let status = res.status(); let headers = res.headers().clone(); - let response = body::to_bytes(res.into_body()).await.unwrap().to_vec(); - let actual_body = str::from_utf8(&response).unwrap().to_string(); + let body = res.bytes().await?; + let actual_body = str::from_utf8(&body).unwrap().to_string(); ensure_eq!( expected, @@ -55,26 +53,15 @@ pub async fn assert_http_response( Ok(()) } -pub async fn create_request(method: Method, url: &str, body: &str) -> Result> { - let req = Request::builder() - .method(method) - .uri(url) - .body(Body::from(body.to_string())) - .expect("request builder"); +pub async fn create_request(method: Method, url: &str, body: &str) -> Result { + let mut req = reqwest::Request::new(method, url.try_into()?); + *req.body_mut() = Some(body.to_owned().into()); Ok(req) } -pub fn create_client() -> Client> { - let connector = HttpsConnector::new(); - Client::builder() - .pool_max_idle_per_host(0) - .build::<_, hyper::Body>(connector) -} - -pub async fn make_request(method: Method, path: &str, body: &str) -> Result> { - let c = create_client(); - let req = create_request(method, path, body); - let resp = c.request(req.await?).await.unwrap(); - Ok(resp) +pub async fn make_request(method: Method, path: &str, body: &str) -> Result { + let req = create_request(method, path, body).await?; + let client = reqwest::Client::new(); + Ok(client.execute(req).await?) } diff --git a/crates/llm-local/src/lib.rs b/crates/llm-local/src/lib.rs index aa9d1355b0..60e1b7c02a 100644 --- a/crates/llm-local/src/lib.rs +++ b/crates/llm-local/src/lib.rs @@ -11,7 +11,7 @@ use llm::{ use rand::SeedableRng; use spin_core::async_trait; use spin_llm::{LlmEngine, MODEL_ALL_MINILM_L6_V2}; -use spin_world::v1::llm::{self as wasi_llm}; +use spin_world::v2::llm::{self as wasi_llm}; use std::{ collections::hash_map::Entry, collections::HashMap, diff --git a/crates/llm-remote-http/src/lib.rs b/crates/llm-remote-http/src/lib.rs index 6110dc8979..15c29baf60 100644 --- a/crates/llm-remote-http/src/lib.rs +++ b/crates/llm-remote-http/src/lib.rs @@ -7,7 +7,7 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use spin_core::async_trait; use spin_llm::LlmEngine; -use spin_world::v1::llm::{self as wasi_llm}; +use spin_world::v2::llm::{self as wasi_llm}; #[derive(Clone)] pub struct RemoteHttpLlmEngine { diff --git a/crates/llm/src/host_component.rs b/crates/llm/src/host_component.rs index 62d9476b0b..8574e6bb0e 100644 --- a/crates/llm/src/host_component.rs +++ b/crates/llm/src/host_component.rs @@ -25,7 +25,8 @@ impl HostComponent for LlmComponent { linker: &mut spin_core::Linker, get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, ) -> anyhow::Result<()> { - spin_world::v1::llm::add_to_linker(linker, get) + spin_world::v1::llm::add_to_linker(linker, get)?; + spin_world::v2::llm::add_to_linker(linker, get) } fn build_data(&self) -> Self::Data { diff --git a/crates/llm/src/lib.rs b/crates/llm/src/lib.rs index 2d12769497..f322fe01dd 100644 --- a/crates/llm/src/lib.rs +++ b/crates/llm/src/lib.rs @@ -2,7 +2,8 @@ pub mod host_component; use spin_app::MetadataKey; use spin_core::async_trait; -use spin_world::v1::llm::{self as wasi_llm}; +use spin_world::v1::llm::{self as v1}; +use spin_world::v2::llm::{self as v2}; use std::collections::HashSet; pub use crate::host_component::LlmComponent; @@ -14,16 +15,16 @@ pub const AI_MODELS_KEY: MetadataKey> = MetadataKey::new("ai_mod pub trait LlmEngine: Send + Sync { async fn infer( &mut self, - model: wasi_llm::InferencingModel, + model: v1::InferencingModel, prompt: String, - params: wasi_llm::InferencingParams, - ) -> Result; + params: v2::InferencingParams, + ) -> Result; async fn generate_embeddings( &mut self, - model: wasi_llm::EmbeddingModel, + model: v2::EmbeddingModel, data: Vec, - ) -> Result; + ) -> Result; } pub struct LlmDispatch { @@ -32,13 +33,13 @@ pub struct LlmDispatch { } #[async_trait] -impl wasi_llm::Host for LlmDispatch { +impl v2::Host for LlmDispatch { async fn infer( &mut self, - model: wasi_llm::InferencingModel, + model: v2::InferencingModel, prompt: String, - params: Option, - ) -> anyhow::Result> { + params: Option, + ) -> anyhow::Result> { if !self.allowed_models.contains(&model) { return Ok(Err(access_denied_error(&model))); } @@ -47,7 +48,7 @@ impl wasi_llm::Host for LlmDispatch { .infer( model, prompt, - params.unwrap_or(wasi_llm::InferencingParams { + params.unwrap_or(v2::InferencingParams { max_tokens: 100, repeat_penalty: 1.1, repeat_penalty_last_n_token_count: 64, @@ -61,9 +62,9 @@ impl wasi_llm::Host for LlmDispatch { async fn generate_embeddings( &mut self, - m: wasi_llm::EmbeddingModel, + m: v1::EmbeddingModel, data: Vec, - ) -> anyhow::Result> { + ) -> anyhow::Result> { if !self.allowed_models.contains(&m) { return Ok(Err(access_denied_error(&m))); } @@ -71,8 +72,36 @@ impl wasi_llm::Host for LlmDispatch { } } -fn access_denied_error(model: &str) -> wasi_llm::Error { - wasi_llm::Error::InvalidInput(format!( +#[async_trait] +impl v1::Host for LlmDispatch { + async fn infer( + &mut self, + model: v1::InferencingModel, + prompt: String, + params: Option, + ) -> anyhow::Result> { + Ok( + ::infer(self, model, prompt, params.map(Into::into)) + .await? + .map(Into::into) + .map_err(Into::into), + ) + } + + async fn generate_embeddings( + &mut self, + model: v1::EmbeddingModel, + data: Vec, + ) -> anyhow::Result> { + Ok(::generate_embeddings(self, model, data) + .await? + .map(Into::into) + .map_err(Into::into)) + } +} + +fn access_denied_error(model: &str) -> v2::Error { + v2::Error::InvalidInput(format!( "The component does not have access to use '{model}'. To give the component access, add '{model}' to the 'ai_models' key for the component in your spin.toml manifest" )) } diff --git a/crates/trigger/src/runtime_config/llm.rs b/crates/trigger/src/runtime_config/llm.rs index bead39107c..af5f2a3629 100644 --- a/crates/trigger/src/runtime_config/llm.rs +++ b/crates/trigger/src/runtime_config/llm.rs @@ -1,7 +1,7 @@ use async_trait::async_trait; use spin_llm::LlmEngine; use spin_llm_remote_http::RemoteHttpLlmEngine; -use spin_world::v1::llm as wasi_llm; +use spin_world::v2::llm as wasi_llm; use url::Url; #[derive(Default)] diff --git a/crates/world/src/conversions.rs b/crates/world/src/conversions.rs index f2b068b14e..5430f17860 100644 --- a/crates/world/src/conversions.rs +++ b/crates/world/src/conversions.rs @@ -168,3 +168,53 @@ mod redis { } } } + +mod llm { + use super::*; + + impl From for v2::llm::InferencingParams { + fn from(value: v1::llm::InferencingParams) -> Self { + Self { + max_tokens: value.max_tokens, + repeat_penalty: value.repeat_penalty, + repeat_penalty_last_n_token_count: value.repeat_penalty_last_n_token_count, + temperature: value.temperature, + top_k: value.top_k, + top_p: value.top_p, + } + } + } + + impl From for v1::llm::InferencingResult { + fn from(value: v2::llm::InferencingResult) -> Self { + Self { + text: value.text, + usage: v1::llm::InferencingUsage { + prompt_token_count: value.usage.prompt_token_count, + generated_token_count: value.usage.prompt_token_count, + }, + } + } + } + + impl From for v1::llm::EmbeddingsResult { + fn from(value: v2::llm::EmbeddingsResult) -> Self { + Self { + embeddings: value.embeddings, + usage: v1::llm::EmbeddingsUsage { + prompt_token_count: value.usage.prompt_token_count, + }, + } + } + } + + impl From for v1::llm::Error { + fn from(value: v2::llm::Error) -> Self { + match value { + v2::llm::Error::ModelNotSupported => Self::ModelNotSupported, + v2::llm::Error::RuntimeError(s) => Self::RuntimeError(s), + v2::llm::Error::InvalidInput(s) => Self::InvalidInput(s), + } + } + } +} diff --git a/sdk/rust/src/http.rs b/sdk/rust/src/http.rs index 9a74d134d6..ae166fbdaf 100644 --- a/sdk/rust/src/http.rs +++ b/sdk/rust/src/http.rs @@ -262,7 +262,8 @@ impl Response { ResponseBuilder { response: self } } - fn builder() -> ResponseBuilder { + /// Creates a [`ResponseBuilder`] + pub fn builder() -> ResponseBuilder { ResponseBuilder::new(200) } } diff --git a/tests/spinup_tests.rs b/tests/spinup_tests.rs index 6cd484fa64..1943827f0e 100644 --- a/tests/spinup_tests.rs +++ b/tests/spinup_tests.rs @@ -103,6 +103,11 @@ mod spinup_tests { testcases::head_rust_sdk_redis(CONTROLLER).await } + #[tokio::test] + async fn llm_works() { + testcases::llm_works(CONTROLLER).await + } + #[tokio::test] async fn header_env_routes_works() { testcases::header_env_routes_works(CONTROLLER).await diff --git a/tests/testcases/llm/Cargo.lock b/tests/testcases/llm/Cargo.lock new file mode 100644 index 0000000000..afea20d9a5 --- /dev/null +++ b/tests/testcases/llm/Cargo.lock @@ -0,0 +1,582 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "ai" +version = "0.1.0" +dependencies = [ + "anyhow", + "spin-sdk", +] + +[[package]] +name = "anyhow" +version = "1.0.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854" + +[[package]] +name = "async-trait" +version = "0.1.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a66537f1bb974b254c98ed142ff995236e81b9d0fe4db0575f46612cb15eb0f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4682ae6287fcf752ecaabbfcc7b6f9b72aa33933dc23a554d853aea8eea8635" + +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "form_urlencoded" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652" +dependencies = [ + "percent-encoding", +] + +[[package]] +name = "futures" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da0290714b38af9b4a7b094b8a37086d1b4e61f2df9122c3cad2577669145335" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" + +[[package]] +name = "futures-executor" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f4fb8693db0cf099eadcca0efe2a5a22e4550f98ed16aba6c48700da29597bc" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" + +[[package]] +name = "futures-macro" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "futures-sink" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" + +[[package]] +name = "futures-task" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" + +[[package]] +name = "futures-util" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + +[[package]] +name = "hashbrown" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93e7192158dbcda357bdec5fb5788eebf8bbac027f3f33e719d29135ae84156" + +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" +dependencies = [ + "unicode-segmentation", +] + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "id-arena" +version = "2.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "25a2bc672d1148e28034f176e01fffebb08b35768468cc954630da77a1449005" + +[[package]] +name = "indexmap" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8adf3ddd720272c6ea8bf59463c04e0f93d0bbf7c5439b691bca2987e0270897" +dependencies = [ + "equivalent", + "hashbrown", + "serde", +] + +[[package]] +name = "itoa" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af150ab688ff2122fcef229be89cb50dd66af9e01a4ff320cc137eecc9bacc38" + +[[package]] +name = "leb128" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "884e2677b40cc8c339eaefcb701c32ef1fd2493d71118dc0ca4b6a736c93bd67" + +[[package]] +name = "log" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" + +[[package]] +name = "memchr" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" + +[[package]] +name = "once_cell" +version = "1.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" + +[[package]] +name = "percent-encoding" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "proc-macro2" +version = "1.0.66" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18fb31db3f9bddb2ea821cde30a9f70117e3f119938b5ee630b7403aa6e2ead9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50f3b39ccfb720540debaa0164757101c08ecb8d326b15358ce76a62c7e85965" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "routefinder" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94f8f99b10dedd317514253dda1fa7c14e344aac96e1f78149a64879ce282aca" +dependencies = [ + "smartcow", + "smartstring", +] + +[[package]] +name = "ryu" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad4cc8da4ef723ed60bced201181d83791ad433213d8c24efffda1eec85d741" + +[[package]] +name = "semver" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0293b4b29daaf487284529cc2f5675b8e57c61f70167ba415a463651fd6a918" + +[[package]] +name = "serde" +version = "1.0.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" + +[[package]] +name = "serde_derive" +version = "1.0.183" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "serde_json" +version = "1.0.108" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d1c7e3eac408d115102c4c24ad393e0821bb3a5df4d506a80f85f7a742a526b" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + +[[package]] +name = "smallvec" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" + +[[package]] +name = "smartcow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "656fcb1c1fca8c4655372134ce87d8afdf5ec5949ebabe8d314be0141d8b5da2" +dependencies = [ + "smartstring", +] + +[[package]] +name = "smartstring" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb72c633efbaa2dd666986505016c32c3044395ceaf881518399d2f4127ee29" +dependencies = [ + "autocfg", + "static_assertions", + "version_check", +] + +[[package]] +name = "spdx" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b19b32ed6d899ab23174302ff105c1577e45a06b08d4fe0a9dd13ce804bbbf71" +dependencies = [ + "smallvec", +] + +[[package]] +name = "spin-macro" +version = "0.1.0" +dependencies = [ + "anyhow", + "bytes", + "http", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "spin-sdk" +version = "2.0.0-pre0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "form_urlencoded", + "futures", + "http", + "once_cell", + "routefinder", + "serde", + "serde_json", + "spin-macro", + "thiserror", + "wit-bindgen", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04361975b3f5e348b2189d8dc55bc942f278b2d482a6a0365de5bdd62d351567" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "611040a08a0439f8248d1990b111c95baa9c704c805fa1f62104b39655fd7f90" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "090198534930841fab3a5d1bb637cde49e339654e606195f8d9c76eeb081dc96" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.28", +] + +[[package]] +name = "unicode-ident" +version = "1.0.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "301abaae475aa91687eb82514b328ab47a211a533026cb25fc3e519b86adfc3c" + +[[package]] +name = "unicode-segmentation" +version = "1.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36" + +[[package]] +name = "unicode-xid" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "wasm-encoder" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53ae0be20bf87918df4fa831bfbbd0b491d24aee407ed86360eae4c2c5608d38" +dependencies = [ + "leb128", +] + +[[package]] +name = "wasm-metadata" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5621910462c61a8efc3248fdfb1739bf649bb335b0df935c27b340418105f9d8" +dependencies = [ + "anyhow", + "indexmap", + "serde", + "serde_derive", + "serde_json", + "spdx", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.116.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53290b1276c5c2d47d694fb1a920538c01f51690e7e261acbe1d10c5fc306ea1" +dependencies = [ + "indexmap", + "semver", +] + +[[package]] +name = "wit-bindgen" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38726c54a5d7c03cac28a2a8de1006cfe40397ddf6def3f836189033a413bc08" +dependencies = [ + "bitflags", + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8bf1fddccaff31a1ad57432d8bfb7027a7e552969b6c68d6d8820dcf5c2371f" +dependencies = [ + "anyhow", + "wit-component", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7200e565124801e01b7b5ddafc559e1da1b2e1bed5364d669cd1d96fb88722" +dependencies = [ + "anyhow", + "heck", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ae33920ad8119fe72cf59eb00f127c0b256a236b9de029a1a10397b1f38bdbd" +dependencies = [ + "anyhow", + "proc-macro2", + "quote", + "syn 2.0.28", + "wit-bindgen-core", + "wit-bindgen-rust", + "wit-component", +] + +[[package]] +name = "wit-component" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "480cc1a078b305c1b8510f7c455c76cbd008ee49935f3a6c5fd5e937d8d95b1e" +dependencies = [ + "anyhow", + "bitflags", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43771ee863a16ec4ecf9da0fc65c3bbd4a1235c8e3da5f094b562894843dfa76" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", +] diff --git a/tests/testcases/llm/Cargo.toml b/tests/testcases/llm/Cargo.toml new file mode 100644 index 0000000000..c7d87f5f05 --- /dev/null +++ b/tests/testcases/llm/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "ai" +authors = ["Ryan Levick "] +description = "" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["cdylib"] + +[dependencies] +anyhow = "1" +spin-sdk = { path = "../../../sdk/rust" } + +[workspace] diff --git a/tests/testcases/llm/spin.toml b/tests/testcases/llm/spin.toml new file mode 100644 index 0000000000..101d5c10d4 --- /dev/null +++ b/tests/testcases/llm/spin.toml @@ -0,0 +1,17 @@ +spin_manifest_version = "1" +authors = ["Ryan Levick "] +description = "" +name = "ai" +trigger = { type = "http", base = "/" } +version = "0.1.0" + +[[component]] +id = "ai" +source = "target/wasm32-wasi/release/ai.wasm" +allowed_http_hosts = [] +ai_models = ["llama2-chat","all-minilm-l6-v2"] +[component.trigger] +route = "/..." +[component.build] +command = "cargo build --target wasm32-wasi --release" +watch = ["src/**/*.rs", "Cargo.toml"] diff --git a/tests/testcases/llm/src/lib.rs b/tests/testcases/llm/src/lib.rs new file mode 100644 index 0000000000..ad40acbb4f --- /dev/null +++ b/tests/testcases/llm/src/lib.rs @@ -0,0 +1,14 @@ +use anyhow::Result; +use spin_sdk::{ + http::{Request, Response}, + http_component, llm, +}; + +/// A simple Spin HTTP component. +#[http_component] +fn hello_world(_req: Request) -> Result { + let model = llm::InferencingModel::Llama2Chat; + let inference = llm::infer(model, "say hello")?; + + Ok(Response::builder().status(200).body(inference.text).build()) +} diff --git a/tests/testcases/mod.rs b/tests/testcases/mod.rs index 59edc2b863..121cf94fe6 100644 --- a/tests/testcases/mod.rs +++ b/tests/testcases/mod.rs @@ -845,6 +845,45 @@ pub async fn head_rust_sdk_redis(controller: &dyn Controller) { tc.run(controller).await.unwrap() } +pub async fn llm_works(controller: &dyn Controller) { + async fn checks( + metadata: AppMetadata, + _: Option>>, + _: Option>>, + ) -> Result<()> { + let response = e2e_testing::http_asserts::make_request( + Method::GET, + get_url(metadata.base.as_str(), "/").as_str(), + "", + ) + .await?; + // We avoid actually running inferencing because it's slow and instead just + // ensure that the app boots properly. + assert_eq!(response.status(), 500); + let body = std::str::from_utf8(&response.bytes().await?) + .unwrap() + .to_string(); + assert!(body.contains("Could not read model registry directory")); + + Ok(()) + } + + let tc = TestCaseBuilder::default() + .name("llm".to_string()) + .appname(Some("llm".to_string())) + .assertions( + |metadata: AppMetadata, + stdout_stream: Option>>, + stderr_stream: Option>>| { + Box::pin(checks(metadata, stdout_stream, stderr_stream)) + }, + ) + .build() + .unwrap(); + + tc.run(controller).await.unwrap() +} + pub async fn header_env_routes_works(controller: &dyn Controller) { async fn checks( metadata: AppMetadata,