From 0de1d494a40452ab370bad9ec22f601b32657616 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 15:39:30 -0400 Subject: [PATCH 01/28] Replace log with tracing in spin-redis Signed-off-by: Lann Martin --- Cargo.lock | 1 - crates/redis/Cargo.toml | 1 - crates/redis/src/lib.rs | 20 +++++++++++--------- crates/redis/src/spin.rs | 6 +++--- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 47ed72136..c6d649d26 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3503,7 +3503,6 @@ dependencies = [ "anyhow", "async-trait", "futures", - "log", "redis", "spin-engine", "spin-manifest", diff --git a/crates/redis/Cargo.toml b/crates/redis/Cargo.toml index 434c49d4f..88b959d71 100644 --- a/crates/redis/Cargo.toml +++ b/crates/redis/Cargo.toml @@ -11,7 +11,6 @@ doctest = false anyhow = "1.0" async-trait = "0.1" futures = "0.3" -log = { version = "0.4", default-features = false } spin-engine = { path = "../engine" } spin-manifest = { path = "../manifest" } spin-trigger = { path = "../trigger" } diff --git a/crates/redis/src/lib.rs b/crates/redis/src/lib.rs index a881eec45..c9985b461 100644 --- a/crates/redis/src/lib.rs +++ b/crates/redis/src/lib.rs @@ -2,7 +2,8 @@ mod spin; -use crate::spin::SpinRedisExecutor; +use std::{collections::HashMap, sync::Arc}; + use anyhow::Result; use async_trait::async_trait; use futures::StreamExt; @@ -10,7 +11,8 @@ use redis::{Client, ConnectionLike}; use spin_manifest::{ComponentMap, RedisConfig, RedisTriggerConfiguration, TriggerConfig}; use spin_redis::SpinRedisData; use spin_trigger::{cli::NoArgs, TriggerExecutor}; -use std::{collections::HashMap, sync::Arc}; + +use crate::spin::SpinRedisExecutor; wit_bindgen_wasmtime::import!({paths: ["../../wit/ephemeral/spin-redis.wit"], async: *}); @@ -80,14 +82,14 @@ impl TriggerExecutor for RedisTrigger { async fn run(self, _config: Self::RunConfig) -> Result<()> { let address = self.trigger_config.address.as_str(); - log::info!("Connecting to Redis server at {}", address); + tracing::info!("Connecting to Redis server at {}", address); let mut client = Client::open(address.to_string())?; let mut pubsub = client.get_async_connection().await?.into_pubsub(); // Subscribe to channels for (subscription, idx) in self.subscriptions.iter() { let name = &self.engine.config.components[*idx].id; - log::info!( + tracing::info!( "Subscribed component #{} ({}) to channel: {}", idx, name, @@ -101,9 +103,9 @@ impl TriggerExecutor for RedisTrigger { match stream.next().await { Some(msg) => drop(self.handle(msg).await), None => { - log::trace!("Empty message"); + tracing::trace!("Empty message"); if !client.check_connection() { - log::info!("No Redis connection available"); + tracing::info!("No Redis connection available"); break Ok(()); } } @@ -116,7 +118,7 @@ impl RedisTrigger { // Handle the message. async fn handle(&self, msg: redis::Msg) -> Result<()> { let channel = msg.get_channel_name(); - log::info!("Received message on channel {:?}", channel); + tracing::info!("Received message on channel {:?}", channel); if let Some(idx) = self.subscriptions.get(channel).copied() { let component = &self.engine.config.components[idx]; @@ -134,7 +136,7 @@ impl RedisTrigger { match executor { spin_manifest::RedisExecutor::Spin => { - log::trace!("Executing Spin Redis component {}", component.id); + tracing::trace!("Executing Spin Redis component {}", component.id); let executor = SpinRedisExecutor; executor .execute( @@ -148,7 +150,7 @@ impl RedisTrigger { } }; } else { - log::debug!("No subscription found for {:?}", channel); + tracing::debug!("No subscription found for {:?}", channel); } Ok(()) diff --git a/crates/redis/src/spin.rs b/crates/redis/src/spin.rs index 6fe77f66e..d53c22ffb 100644 --- a/crates/redis/src/spin.rs +++ b/crates/redis/src/spin.rs @@ -17,7 +17,7 @@ impl RedisExecutor for SpinRedisExecutor { payload: &[u8], follow: bool, ) -> Result<()> { - log::trace!( + tracing::trace!( "Executing request using the Spin executor for component {}", component ); @@ -30,11 +30,11 @@ impl RedisExecutor for SpinRedisExecutor { let result = match Self::execute_impl(store, instance, channel, payload.to_vec()).await { Ok(()) => { - log::trace!("Request finished OK"); + tracing::trace!("Request finished OK"); Ok(()) } Err(e) => { - log::trace!("Request finished with error {}", e); + tracing::trace!("Request finished with error {}", e); Err(e) } }; From 216a5f7a2325607358ed05c7d21e378102ae89dd Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 09:55:03 -0400 Subject: [PATCH 02/28] Copy spin-core and spin-app crates from prototype Signed-off-by: Lann Martin --- Cargo.lock | 81 +++++++++ Cargo.toml | 2 + crates/app/Cargo.toml | 13 ++ crates/app/src/host_component.rs | 51 ++++++ crates/app/src/lib.rs | 268 ++++++++++++++++++++++++++++++ crates/app/src/locked.rs | 154 +++++++++++++++++ crates/app/src/values.rs | 57 +++++++ crates/core/Cargo.toml | 13 ++ crates/core/src/host_component.rs | 129 ++++++++++++++ crates/core/src/io.rs | 16 ++ crates/core/src/lib.rs | 173 +++++++++++++++++++ crates/core/src/limits.rs | 35 ++++ crates/core/src/store.rs | 234 ++++++++++++++++++++++++++ 13 files changed, 1226 insertions(+) create mode 100644 crates/app/Cargo.toml create mode 100644 crates/app/src/host_component.rs create mode 100644 crates/app/src/lib.rs create mode 100644 crates/app/src/locked.rs create mode 100644 crates/app/src/values.rs create mode 100644 crates/core/Cargo.toml create mode 100644 crates/core/src/host_component.rs create mode 100644 crates/core/src/io.rs create mode 100644 crates/core/src/lib.rs create mode 100644 crates/core/src/limits.rs create mode 100644 crates/core/src/store.rs diff --git a/Cargo.lock b/Cargo.lock index c6d649d26..fab77f082 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" + [[package]] name = "addr2line" version = "0.17.0" @@ -37,6 +43,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "aliasable" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "250f629c0161ad8107cf89319e990051fae62832fd343083bea452d93e2205fd" + [[package]] name = "ambient-authority" version = "0.0.1" @@ -2183,6 +2195,29 @@ version = "6.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ff7415e9ae3fff1225851df9e0d9e4e5479f947619774677a63572e55e80eff" +[[package]] +name = "ouroboros" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfbb50b356159620db6ac971c6d5c9ab788c9cc38a6f49619fca2a27acb062ca" +dependencies = [ + "aliasable", + "ouroboros_macro", +] + +[[package]] +name = "ouroboros_macro" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0d9d1a6191c4f391f87219d1ea42b23f09ee84d64763cd05ee6ea88d9f384d" +dependencies = [ + "Inflector", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "outbound-http" version = "0.2.0" @@ -3268,6 +3303,19 @@ dependencies = [ "wit-bindgen-wasmtime", ] +[[package]] +name = "spin-app" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "ouroboros", + "serde", + "serde_json", + "spin-core", + "thiserror", +] + [[package]] name = "spin-build" version = "0.2.0" @@ -3349,6 +3397,19 @@ dependencies = [ "wit-bindgen-wasmtime", ] +[[package]] +name = "spin-core" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "tracing", + "wasi-cap-std-sync", + "wasi-common", + "wasmtime", + "wasmtime-wasi", +] + [[package]] name = "spin-engine" version = "0.2.0" @@ -4312,6 +4373,24 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "wasi-tokio" +version = "0.39.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ab325bba31ae9286b8ebdc18d32a43d6471312c9bc4e477240be444e00ec4f4" +dependencies = [ + "anyhow", + "cap-std 0.25.2", + "io-extras 0.15.0", + "io-lifetimes 0.7.3", + "lazy_static", + "rustix 0.35.9", + "tokio", + "wasi-cap-std-sync", + "wasi-common", + "wiggle", +] + [[package]] name = "wasm-bindgen" version = "0.2.82" @@ -4590,6 +4669,7 @@ dependencies = [ "anyhow", "wasi-cap-std-sync", "wasi-common", + "wasi-tokio", "wasmtime", "wiggle", ] @@ -4677,6 +4757,7 @@ dependencies = [ "tracing", "wasmtime", "wiggle-macro", + "witx", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 5b7736c6a..afd00c2c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,8 +71,10 @@ e2e-tests = [] [workspace] members = [ "crates/abi-conformance", + "crates/app", "crates/build", "crates/config", + "crates/core", "crates/engine", "crates/http", "crates/loader", diff --git a/crates/app/Cargo.toml b/crates/app/Cargo.toml new file mode 100644 index 000000000..1d8dedf4b --- /dev/null +++ b/crates/app/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "spin-app" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +ouroboros = "0.15" +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +spin-core = { path = "../core" } +thiserror = "1.0" diff --git a/crates/app/src/host_component.rs b/crates/app/src/host_component.rs new file mode 100644 index 000000000..c428aefe4 --- /dev/null +++ b/crates/app/src/host_component.rs @@ -0,0 +1,51 @@ +use std::sync::Arc; + +use spin_core::{EngineBuilder, HostComponent, HostComponentsData}; + +use crate::AppComponent; + +pub trait DynamicHostComponent: HostComponent { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()>; +} + +impl DynamicHostComponent for Arc { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { + (**self).update_data(data, component) + } +} + +type DataUpdater = + Box anyhow::Result<()> + Send + Sync>; + +#[derive(Default)] +pub struct DynamicHostComponents { + data_updaters: Vec, +} + +impl DynamicHostComponents { + pub fn add_dynamic_host_component( + &mut self, + engine_builder: &mut EngineBuilder, + host_component: DHC, + ) -> anyhow::Result<()> { + let host_component = Arc::new(host_component); + let handle = engine_builder.add_host_component(host_component.clone())?; + self.data_updaters + .push(Box::new(move |host_components_data, component| { + let data = host_components_data.get_or_insert(handle); + host_component.update_data(data, component) + })); + Ok(()) + } + + pub fn update_data( + &self, + host_components_data: &mut HostComponentsData, + component: &AppComponent, + ) -> anyhow::Result<()> { + for data_updater in &self.data_updaters { + data_updater(host_components_data, component)?; + } + Ok(()) + } +} diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs new file mode 100644 index 000000000..89b099bdd --- /dev/null +++ b/crates/app/src/lib.rs @@ -0,0 +1,268 @@ +mod host_component; +pub mod locked; +pub mod values; + +use ouroboros::self_referencing; +use serde::Deserialize; +use spin_core::{wasmtime, Engine, EngineBuilder, StoreBuilder}; + +use host_component::DynamicHostComponents; +use locked::{ContentPath, LockedApp, LockedComponent, LockedComponentSource, LockedTrigger}; + +pub use async_trait::async_trait; +pub use host_component::DynamicHostComponent; +pub use locked::Variable; + +// TODO(lann): Should this migrate to spin-loader? +#[async_trait] +pub trait Loader { + async fn load_app(&self, uri: &str) -> anyhow::Result; + + async fn load_module( + &self, + engine: &wasmtime::Engine, + source: &LockedComponentSource, + ) -> anyhow::Result; + + async fn mount_files( + &self, + store_builder: &mut StoreBuilder, + component: &AppComponent, + ) -> anyhow::Result<()>; +} + +pub struct AppLoader { + inner: Box, + dynamic_host_components: DynamicHostComponents, +} + +impl AppLoader { + pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { + Self { + inner: Box::new(loader), + dynamic_host_components: Default::default(), + } + } + + pub fn add_dynamic_host_component( + &mut self, + engine_builder: &mut EngineBuilder, + host_component: DHC, + ) -> anyhow::Result<()> { + self.dynamic_host_components + .add_dynamic_host_component(engine_builder, host_component) + } + + pub async fn load_app(&self, uri: String) -> Result { + let locked = self + .inner + .load_app(&uri) + .await + .map_err(Error::LoaderError)?; + Ok(App { + loader: self, + uri, + locked, + }) + } + + pub async fn load_owned_app(self, uri: String) -> Result { + OwnedApp::try_new_async(self, |loader| Box::pin(loader.load_app(uri))).await + } +} + +impl std::fmt::Debug for AppLoader { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("AppLoader").finish() + } +} + +#[self_referencing] +#[derive(Debug)] +pub struct OwnedApp { + loader: AppLoader, + + #[borrows(loader)] + #[covariant] + app: App<'this>, +} + +impl std::ops::Deref for OwnedApp { + type Target = App<'static>; + + fn deref(&self) -> &Self::Target { + unsafe { + // We know that App's lifetime param is for AppLoader, which is owned by self here. + std::mem::transmute::<&App, &App<'static>>(self.borrow_app()) + } + } +} + +#[derive(Debug)] +pub struct App<'a> { + loader: &'a AppLoader, + uri: String, + locked: LockedApp, +} + +impl<'a> App<'a> { + pub fn uri(&self) -> &str { + &self.uri + } + + pub fn get_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Option> { + self.locked + .metadata + .get(key) + .map(|value| Ok(T::deserialize(value)?)) + } + + pub fn require_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result { + self.get_metadata(key) + .ok_or_else(|| Error::ManifestError(format!("missing required {key:?}")))? + } + + pub fn variables(&self) -> impl Iterator { + self.locked.variables.iter() + } + + pub fn components(&self) -> impl Iterator { + self.locked + .components + .iter() + .map(|locked| AppComponent { app: self, locked }) + } + + pub fn get_component(&self, component_id: &str) -> Option { + self.components() + .find(|component| component.locked.id == component_id) + } + + pub fn triggers(&self) -> impl Iterator { + self.locked + .triggers + .iter() + .map(|locked| AppTrigger { app: self, locked }) + } + + pub fn triggers_with_type(&'a self, trigger_type: &'a str) -> impl Iterator { + self.triggers() + .filter(move |trigger| trigger.locked.trigger_type == trigger_type) + } +} + +pub struct AppComponent<'a> { + pub app: &'a App<'a>, + locked: &'a LockedComponent, +} + +impl<'a> AppComponent<'a> { + pub fn id(&self) -> &str { + &self.locked.id + } + + pub fn source(&self) -> &LockedComponentSource { + &self.locked.source + } + + pub fn files(&self) -> std::slice::Iter { + self.locked.files.iter() + } + + pub fn get_metadata>(&self, key: &str) -> Option> { + self.locked + .metadata + .get(key) + .map(|value| Ok(T::deserialize(value)?)) + } + + pub fn config(&self) -> impl Iterator { + self.locked.config.iter() + } + + pub async fn load_module( + &self, + engine: &Engine, + ) -> Result { + self.app + .loader + .inner + .load_module(engine.as_ref(), &self.locked.source) + .await + .map_err(Error::LoaderError) + } + + pub async fn apply_store_config(&self, builder: &mut StoreBuilder) -> Result<()> { + builder.env(&self.locked.env).map_err(Error::CoreError)?; + + let loader = self.app.loader; + loader + .inner + .mount_files(builder, self) + .await + .map_err(Error::LoaderError)?; + + loader + .dynamic_host_components + .update_data(builder.host_components_data(), self) + .map_err(Error::HostComponentError)?; + + Ok(()) + } +} + +pub struct AppTrigger<'a> { + pub app: &'a App<'a>, + locked: &'a LockedTrigger, +} + +impl<'a> AppTrigger<'a> { + pub fn id(&self) -> &str { + &self.locked.id + } + + pub fn trigger_type(&self) -> &str { + &self.locked.trigger_type + } + + pub fn component(&self) -> Result> { + let component_id = self.locked.trigger_config.get("component").ok_or_else(|| { + Error::ManifestError(format!( + "trigger {:?} missing 'component' config field", + self.locked.id + )) + })?; + let component_id = component_id.as_str().ok_or_else(|| { + Error::ManifestError(format!( + "trigger {:?} 'component' field has unexpected value {:?}", + self.locked.id, component_id + )) + })?; + self.app.get_component(component_id).ok_or_else(|| { + Error::ManifestError(format!( + "missing component {:?} configured for trigger {:?}", + component_id, self.locked.id + )) + }) + } + + pub fn typed_config>(&self) -> Result { + Ok(Config::deserialize(&self.locked.trigger_config)?) + } +} + +pub type Result = std::result::Result; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + #[error("spin core error: {0}")] + CoreError(anyhow::Error), + #[error("host component error: {0}")] + HostComponentError(anyhow::Error), + #[error("loader error: {0}")] + LoaderError(anyhow::Error), + #[error("manifest error: {0}")] + ManifestError(String), + #[error("json error: {0}")] + JsonError(#[from] serde_json::Error), +} diff --git a/crates/app/src/locked.rs b/crates/app/src/locked.rs new file mode 100644 index 000000000..83c90bebb --- /dev/null +++ b/crates/app/src/locked.rs @@ -0,0 +1,154 @@ +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::values::ValuesMap; + +// LockedMap gives deterministic encoding, which we want. +pub type LockedMap = std::collections::BTreeMap; + +/// A LockedApp represents a "fully resolved" Spin application. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedApp { + /// Locked schema version + pub spin_lock_version: FixedVersion<0>, + /// Application metadata + #[serde(default, skip_serializing_if = "ValuesMap::is_empty")] + pub metadata: ValuesMap, + /// Custom config variables + #[serde(default, skip_serializing_if = "LockedMap::is_empty")] + pub variables: LockedMap, + /// Application triggers + pub triggers: Vec, + /// Application components + pub components: Vec, +} + +impl LockedApp { + pub fn from_json(contents: &[u8]) -> serde_json::Result { + serde_json::from_slice(contents) + } + + pub fn to_json(&self) -> serde_json::Result> { + serde_json::to_vec_pretty(&self) + } +} + +/// A LockedComponent represents a "fully resolved" Spin component. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedComponent { + /// Application-unique component identifier + pub id: String, + /// Component metadata + #[serde(default, skip_serializing_if = "ValuesMap::is_empty")] + pub metadata: ValuesMap, + /// Wasm source + pub source: LockedComponentSource, + /// WASI environment variables + #[serde(default, skip_serializing_if = "LockedMap::is_empty")] + pub env: LockedMap, + /// WASI filesystem contents + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub files: Vec, + /// Custom config values + #[serde(default, skip_serializing_if = "LockedMap::is_empty")] + pub config: LockedMap, +} + +/// A LockedComponentSource specifies a Wasm source. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedComponentSource { + /// Wasm source content type (e.g. "application/wasm") + pub content_type: String, + /// Wasm source content specification + #[serde(flatten)] + pub content: ContentRef, +} + +/// A ContentPath specifies content mapped to a WASI path. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct ContentPath { + /// Content specification + #[serde(flatten)] + pub content: ContentRef, + /// WASI mount path + pub path: PathBuf, +} + +/// A ContentRef represents content used by an application. +/// +/// At least one of `source` or `digest` must be specified. Implementations may +/// require one or the other (or both). +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct ContentRef { + /// A URI where the content can be accessed. Implementations may support + /// different URI schemes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source: Option, + /// If set, the content must have the given SHA-256 digest. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub digest: Option, +} + +/// A LockedTrigger specifies configuration for an application trigger. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct LockedTrigger { + /// Application-unique trigger identifier + pub id: String, + /// Trigger type (e.g. "http") + pub trigger_type: String, + /// Trigger-type-specific configuration + pub trigger_config: Value, +} + +/// A Variable specifies a custom configuration variable. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Variable { + /// The variable's default value. If unset, the variable is required. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub default: Option, + /// If set, the variable's value may be sensitive and e.g. shouldn't be logged. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub secret: bool, +} + +/// FixedVersion represents a schema version field with a const value. +#[allow(unused)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[serde(into = "usize", try_from = "usize")] +pub struct FixedVersion; + +impl From> for usize { + fn from(_: FixedVersion) -> usize { + V + } +} + +impl From> for String { + fn from(_: FixedVersion) -> String { + V.to_string() + } +} + +impl TryFrom for FixedVersion { + type Error = String; + + fn try_from(value: usize) -> Result { + if value != V { + return Err(format!("invalid version {} != {}", value, V)); + } + Ok(Self) + } +} + +impl TryFrom for FixedVersion { + type Error = String; + + fn try_from(value: String) -> Result { + let value: usize = value + .parse() + .map_err(|err| format!("invalid version: {}", err))?; + value.try_into() + } +} diff --git a/crates/app/src/values.rs b/crates/app/src/values.rs new file mode 100644 index 000000000..6a764a662 --- /dev/null +++ b/crates/app/src/values.rs @@ -0,0 +1,57 @@ +use serde::Serialize; +use serde_json::Value; + +// ValuesMap stores dynamically-typed values. +pub type ValuesMap = serde_json::Map; + +/// ValuesMapBuilder assists in building a ValuesMap. +#[derive(Default)] +pub struct ValuesMapBuilder(ValuesMap); + +impl ValuesMapBuilder { + pub fn new() -> Self { + Self::default() + } + + pub fn string(&mut self, key: impl Into, value: impl Into) -> &mut Self { + self.entry(key, value.into()) + } + + pub fn string_option( + &mut self, + key: impl Into, + value: Option>, + ) -> &mut Self { + if let Some(value) = value { + self.0.insert(key.into(), value.into().into()); + } + self + } + + pub fn string_array>( + &mut self, + key: impl Into, + iter: impl IntoIterator, + ) -> &mut Self { + self.entry(key, iter.into_iter().map(|s| s.into()).collect::>()) + } + + pub fn entry(&mut self, key: impl Into, value: impl Into) -> &mut Self { + self.0.insert(key.into(), value.into()); + self + } + + pub fn serializable( + &mut self, + key: impl Into, + value: impl Serialize, + ) -> serde_json::Result<&mut Self> { + let value = serde_json::to_value(value)?; + self.0.insert(key.into(), value); + Ok(self) + } + + pub fn build(&mut self) -> ValuesMap { + std::mem::take(&mut self.0) + } +} diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml new file mode 100644 index 000000000..d9308e39f --- /dev/null +++ b/crates/core/Cargo.toml @@ -0,0 +1,13 @@ +[package] +name = "spin-core" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +tracing = "0.1" +async-trait = "0.1" +wasi-cap-std-sync = "0.39" +wasi-common = "0.39" +wasmtime = "0.39" +wasmtime-wasi = { version = "0.39", features = ["tokio"] } \ No newline at end of file diff --git a/crates/core/src/host_component.rs b/crates/core/src/host_component.rs new file mode 100644 index 000000000..6443e290a --- /dev/null +++ b/crates/core/src/host_component.rs @@ -0,0 +1,129 @@ +use std::{any::Any, marker::PhantomData, sync::Arc}; + +use anyhow::Result; + +use super::{Data, Linker}; + +pub trait HostComponent: Send + Sync + 'static { + /// Host component runtime data. + type Data: Send + Sized + 'static; + + /// Add this component to the given Linker, using the given runtime state-getting handle. + // This function signature mirrors those generated by wit-bindgen. + fn add_to_linker( + linker: &mut Linker, + get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> Result<()>; + + fn build_data(&self) -> Self::Data; +} + +impl HostComponent for Arc { + type Data = HC::Data; + + fn add_to_linker( + linker: &mut Linker, + get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> Result<()> { + HC::add_to_linker(linker, get) + } + + fn build_data(&self) -> Self::Data { + (**self).build_data() + } +} + +pub struct HostComponentDataHandle { + idx: usize, + _phantom: PhantomData HC::Data>, +} + +impl Copy for HostComponentDataHandle {} + +impl Clone for HostComponentDataHandle { + fn clone(&self) -> Self { + Self { + idx: self.idx, + _phantom: PhantomData, + } + } +} + +type DataBuilder = Box Box + Send + Sync>; + +pub struct HostComponentsBuilder { + data_builders: Vec, +} + +impl HostComponentsBuilder { + pub fn add_host_component( + &mut self, + linker: &mut Linker, + host_component: HC, + ) -> Result> { + let idx = self.data_builders.len(); + self.data_builders + .push(Box::new(move || Box::new(host_component.build_data()))); + HC::add_to_linker(linker, move |data| { + data.host_components_data + .get_or_insert_idx(idx) + .downcast_mut() + .unwrap() + })?; + Ok(HostComponentDataHandle:: { + idx, + _phantom: PhantomData, + }) + } + + pub fn build(self) -> HostComponents { + let data_builders = Arc::new(self.data_builders); + HostComponents { data_builders } + } +} + +pub struct HostComponents { + data_builders: Arc>, +} + +impl HostComponents { + pub fn builder() -> HostComponentsBuilder { + HostComponentsBuilder { + data_builders: Default::default(), + } + } + + pub fn new_data(&self) -> HostComponentsData { + // Fill with `None` + let data = std::iter::repeat_with(Default::default) + .take(self.data_builders.len()) + .collect(); + HostComponentsData { + data, + data_builders: self.data_builders.clone(), + } + } +} + +pub struct HostComponentsData { + data: Vec>>, + data_builders: Arc>, +} + +impl HostComponentsData { + pub fn get_or_insert( + &mut self, + handle: HostComponentDataHandle, + ) -> &mut HC::Data { + let x = self.get_or_insert_idx(handle.idx); + x.downcast_mut().unwrap() + } + + fn get_or_insert_idx(&mut self, idx: usize) -> &mut Box { + self.data[idx].get_or_insert_with(|| self.data_builders[idx]()) + } + + pub fn set(&mut self, handle: HostComponentDataHandle, data: HC::Data) { + self.data[handle.idx] = Some(Box::new(data)); + } +} diff --git a/crates/core/src/io.rs b/crates/core/src/io.rs new file mode 100644 index 000000000..bf0e6a667 --- /dev/null +++ b/crates/core/src/io.rs @@ -0,0 +1,16 @@ +use std::sync::{Arc, RwLock}; + +use wasi_common::pipe::WritePipe; + +#[derive(Default)] +pub struct OutputBuffer(Arc>>); + +impl OutputBuffer { + pub fn take(&mut self) -> Vec { + std::mem::take(&mut *self.0.write().unwrap()) + } + + pub(crate) fn writer(&self) -> WritePipe> { + WritePipe::from_shared(self.0.clone()) + } +} diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs new file mode 100644 index 000000000..274638b28 --- /dev/null +++ b/crates/core/src/lib.rs @@ -0,0 +1,173 @@ +mod host_component; +mod io; +mod limits; +mod store; + +use std::sync::{Arc, Mutex}; + +use anyhow::Result; +use tracing::instrument; +use wasmtime_wasi::WasiCtx; + +pub use wasmtime::{self, Instance, Module}; + +use self::host_component::{HostComponents, HostComponentsBuilder}; + +pub use host_component::{HostComponent, HostComponentDataHandle, HostComponentsData}; +pub use store::{Store, StoreBuilder}; + +pub struct Config { + inner: wasmtime::Config, +} + +impl Config { + /// Borrow the inner wasmtime::Config mutably. + /// WARNING: This is inherently unstable and may break at any time! + #[doc(hidden)] + pub fn wasmtime_config(&mut self) -> &mut wasmtime::Config { + &mut self.inner + } +} + +impl Default for Config { + fn default() -> Self { + let mut inner = wasmtime::Config::new(); + inner.async_support(true); + Self { inner } + } +} + +pub struct Data { + inner: T, + wasi: WasiCtx, + host_components_data: HostComponentsData, + store_limits: limits::StoreLimitsAsync, +} + +impl AsRef for Data { + fn as_ref(&self) -> &T { + &self.inner + } +} + +impl AsMut for Data { + fn as_mut(&mut self) -> &mut T { + &mut self.inner + } +} + +pub type Linker = wasmtime::Linker>; + +pub struct EngineBuilder { + engine: wasmtime::Engine, + linker: Linker, + host_components_builder: HostComponentsBuilder, +} + +impl EngineBuilder { + fn new(config: &Config) -> Result { + let engine = wasmtime::Engine::new(&config.inner)?; + + let mut linker: Linker = Linker::new(&engine); + wasmtime_wasi::tokio::add_to_linker(&mut linker, |data| &mut data.wasi)?; + + Ok(Self { + engine, + linker, + host_components_builder: HostComponents::builder(), + }) + } + + pub fn link_import( + &mut self, + f: impl FnOnce(&mut Linker, fn(&mut Data) -> &mut T) -> Result<()>, + ) -> Result<()> { + f(&mut self.linker, Data::as_mut) + } + + pub fn add_host_component( + &mut self, + host_component: HC, + ) -> Result> { + self.host_components_builder + .add_host_component(&mut self.linker, host_component) + } + + pub fn build_with_data(self, instance_pre_data: T) -> Engine { + let host_components = self.host_components_builder.build(); + + let instance_pre_store = Arc::new(Mutex::new( + StoreBuilder::new(self.engine.clone(), &host_components) + .build_with_data(instance_pre_data) + .expect("instance_pre_store build should not fail"), + )); + + Engine { + inner: self.engine, + linker: self.linker, + host_components, + instance_pre_store, + } + } +} + +impl EngineBuilder { + pub fn build(self) -> Engine { + self.build_with_data(T::default()) + } +} + +pub struct Engine { + inner: wasmtime::Engine, + linker: Linker, + host_components: HostComponents, + instance_pre_store: Arc>>, +} + +impl Engine { + pub fn builder(config: &Config) -> Result> { + EngineBuilder::new(config) + } + + pub fn store_builder(&self) -> StoreBuilder { + StoreBuilder::new(self.inner.clone(), &self.host_components) + } + + #[instrument(skip_all)] + pub fn instantiate_pre(&self, module: &Module) -> Result> { + let mut store = self.instance_pre_store.lock().unwrap(); + let inner = self.linker.instantiate_pre(&mut *store, module)?; + Ok(InstancePre { inner }) + } +} + +impl AsRef for Engine { + fn as_ref(&self) -> &wasmtime::Engine { + &self.inner + } +} + +pub struct InstancePre { + inner: wasmtime::InstancePre>, +} + +impl InstancePre { + #[instrument(skip_all)] + pub async fn instantiate_async(&self, store: &mut Store) -> Result { + self.inner.instantiate_async(store).await + } +} + +impl Clone for InstancePre { + fn clone(&self) -> Self { + Self { + inner: self.inner.clone(), + } + } +} + +impl AsRef>> for InstancePre { + fn as_ref(&self) -> &wasmtime::InstancePre> { + &self.inner + } +} diff --git a/crates/core/src/limits.rs b/crates/core/src/limits.rs new file mode 100644 index 000000000..28701a5e8 --- /dev/null +++ b/crates/core/src/limits.rs @@ -0,0 +1,35 @@ +use async_trait::async_trait; +use wasmtime::ResourceLimiterAsync; + +/// Async implementation of wasmtime's `StoreLimits`: https://github.com/bytecodealliance/wasmtime/blob/main/crates/wasmtime/src/limits.rs +/// Used to limit the memory use and table size of each Instance +#[derive(Default)] +pub struct StoreLimitsAsync { + max_memory_size: Option, + max_table_elements: Option, +} + +#[async_trait] +impl ResourceLimiterAsync for StoreLimitsAsync { + async fn memory_growing( + &mut self, + _current: usize, + desired: usize, + _maximum: Option, + ) -> bool { + !matches!(self.max_memory_size, Some(limit) if desired > limit) + } + + async fn table_growing(&mut self, _current: u32, desired: u32, _maximum: Option) -> bool { + !matches!(self.max_table_elements, Some(limit) if desired > limit) + } +} + +impl StoreLimitsAsync { + pub fn new(max_memory_size: Option, max_table_elements: Option) -> Self { + Self { + max_memory_size, + max_table_elements, + } + } +} diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs new file mode 100644 index 000000000..9d87a39df --- /dev/null +++ b/crates/core/src/store.rs @@ -0,0 +1,234 @@ +use anyhow::{anyhow, Result}; +use std::{ + io::{Read, Write}, + path::{Path, PathBuf}, +}; +use wasi_cap_std_sync::{ambient_authority, Dir}; +use wasi_common::{dir::DirCaps, pipe::WritePipe, WasiFile}; +use wasi_common::{file::FileCaps, pipe::ReadPipe}; +use wasmtime_wasi::tokio::WasiCtxBuilder; + +use crate::io::OutputBuffer; + +use super::{ + host_component::{HostComponents, HostComponentsData}, + limits::StoreLimitsAsync, + Data, +}; + +pub struct Store { + inner: wasmtime::Store>, +} + +impl Store { + pub fn host_components_data(&mut self) -> &mut HostComponentsData { + &mut self.inner.data_mut().host_components_data + } +} + +impl AsRef>> for Store { + fn as_ref(&self) -> &wasmtime::Store> { + &self.inner + } +} + +impl AsMut>> for Store { + fn as_mut(&mut self) -> &mut wasmtime::Store> { + &mut self.inner + } +} + +impl wasmtime::AsContext for Store { + type Data = Data; + + fn as_context(&self) -> wasmtime::StoreContext<'_, Self::Data> { + self.inner.as_context() + } +} + +impl wasmtime::AsContextMut for Store { + fn as_context_mut(&mut self) -> wasmtime::StoreContextMut<'_, Self::Data> { + self.inner.as_context_mut() + } +} + +// WASI expects preopened dirs in FDs starting at 3 (0-2 are stdio). +const WASI_FIRST_PREOPENED_DIR_FD: u32 = 3; + +const READ_ONLY_DIR_CAPS: DirCaps = DirCaps::from_bits_truncate( + DirCaps::OPEN.bits() + | DirCaps::READDIR.bits() + | DirCaps::READLINK.bits() + | DirCaps::PATH_FILESTAT_GET.bits() + | DirCaps::FILESTAT_GET.bits(), +); +const READ_ONLY_FILE_CAPS: FileCaps = FileCaps::from_bits_truncate( + FileCaps::READ.bits() + | FileCaps::SEEK.bits() + | FileCaps::TELL.bits() + | FileCaps::FILESTAT_GET.bits() + | FileCaps::POLL_READWRITE.bits(), +); + +pub struct StoreBuilder { + engine: wasmtime::Engine, + wasi: std::result::Result, String>, + read_only_preopened_dirs: Vec<(Dir, PathBuf)>, + host_components_data: HostComponentsData, + store_limits: StoreLimitsAsync, +} + +impl StoreBuilder { + pub(crate) fn new(engine: wasmtime::Engine, host_components: &HostComponents) -> Self { + Self { + engine, + wasi: Ok(Some(WasiCtxBuilder::new())), + read_only_preopened_dirs: Vec::new(), + host_components_data: host_components.new_data(), + store_limits: StoreLimitsAsync::default(), + } + } + + pub fn max_memory_size(&mut self, max_memory_size: usize) { + self.store_limits = StoreLimitsAsync::new(Some(max_memory_size), None); + } + + pub fn inherit_stdio(&mut self) { + self.with_wasi(|wasi| wasi.inherit_stdio()); + } + + pub fn stdin(&mut self, file: impl WasiFile + 'static) { + self.with_wasi(|wasi| wasi.stdin(Box::new(file))) + } + + pub fn stdin_pipe(&mut self, r: impl Read + Send + Sync + 'static) { + self.stdin(ReadPipe::new(r)) + } + + pub fn stdout(&mut self, file: impl WasiFile + 'static) { + self.with_wasi(|wasi| wasi.stdout(Box::new(file))) + } + + pub fn stdout_pipe(&mut self, w: impl Write + Send + Sync + 'static) { + self.stdout(WritePipe::new(w)) + } + + pub fn stdout_buffered(&mut self) -> OutputBuffer { + let buffer = OutputBuffer::default(); + self.stdout(buffer.writer()); + buffer + } + + pub fn stderr(&mut self, file: impl WasiFile + 'static) { + self.with_wasi(|wasi| wasi.stderr(Box::new(file))) + } + + pub fn stderr_pipe(&mut self, w: impl Write + Send + Sync + 'static) { + self.stderr(WritePipe::new(w)) + } + + pub fn stderr_buffered(&mut self) -> OutputBuffer { + let buffer = OutputBuffer::default(); + self.stderr(buffer.writer()); + buffer + } + + pub fn args<'b>(&mut self, args: impl IntoIterator) -> Result<()> { + self.try_with_wasi(|mut wasi| { + for arg in args { + wasi = wasi.arg(arg)?; + } + Ok(wasi) + }) + } + + pub fn env( + &mut self, + vars: impl IntoIterator, impl AsRef)>, + ) -> Result<()> { + self.try_with_wasi(|mut wasi| { + for (k, v) in vars { + wasi = wasi.env(k.as_ref(), v.as_ref())?; + } + Ok(wasi) + }) + } + + pub fn read_only_preopened_dir( + &mut self, + host_path: impl AsRef, + guest_path: PathBuf, + ) -> Result<()> { + // WasiCtxBuilder::preopened_dir doesn't let you set capabilities, so we need + // to wait and call WasiCtx::insert_dir after building the WasiCtx. + let dir = wasmtime_wasi::Dir::open_ambient_dir(host_path, ambient_authority())?; + self.read_only_preopened_dirs.push((dir, guest_path)); + Ok(()) + } + + pub fn read_write_preopened_dir( + &mut self, + host_path: impl AsRef, + guest_path: PathBuf, + ) -> Result<()> { + let dir = wasmtime_wasi::Dir::open_ambient_dir(host_path, ambient_authority())?; + self.try_with_wasi(|wasi| wasi.preopened_dir(dir, guest_path)) + } + + pub fn host_components_data(&mut self) -> &mut HostComponentsData { + &mut self.host_components_data + } + + pub fn build_with_data(self, inner_data: T) -> Result> { + let mut wasi = self.wasi.map_err(anyhow::Error::msg)?.unwrap().build(); + + // Insert any read-only preopened dirs + for (idx, (dir, path)) in self.read_only_preopened_dirs.into_iter().enumerate() { + let fd = WASI_FIRST_PREOPENED_DIR_FD + idx as u32; + let dir = Box::new(wasmtime_wasi::tokio::Dir::from_cap_std(dir)); + wasi.insert_dir(fd, dir, READ_ONLY_DIR_CAPS, READ_ONLY_FILE_CAPS, path); + } + + let mut inner = wasmtime::Store::new( + &self.engine, + Data { + inner: inner_data, + wasi, + host_components_data: self.host_components_data, + store_limits: self.store_limits, + }, + ); + inner.limiter_async(move |data| &mut data.store_limits); + Ok(Store { inner }) + } + + pub fn build(self) -> Result> { + self.build_with_data(T::default()) + } + + fn with_wasi(&mut self, f: impl FnOnce(WasiCtxBuilder) -> WasiCtxBuilder) { + let _ = self.try_with_wasi(|wasi| Ok(f(wasi))); + } + + fn try_with_wasi( + &mut self, + f: impl FnOnce(WasiCtxBuilder) -> Result, + ) -> Result<()> { + let wasi = self + .wasi + .as_mut() + .map_err(|err| anyhow!("StoreBuilder already failed: {}", err))? + .take() + .unwrap(); + match f(wasi) { + Ok(wasi) => { + self.wasi = Ok(Some(wasi)); + Ok(()) + } + Err(err) => { + self.wasi = Err(err.to_string()); + Err(err) + } + } + } +} From fa434a904a38d59467f58a6356881cc95ea78bdf Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 10:42:03 -0400 Subject: [PATCH 03/28] Create spin-trigger-new crate This will replace spin-trigger in a future commit. Signed-off-by: Lann Martin --- Cargo.lock | 41 ++++- Cargo.toml | 1 + crates/manifest/src/lib.rs | 14 +- crates/trigger-new/Cargo.toml | 32 ++++ crates/trigger-new/src/cli.rs | 156 +++++++++++++++++ crates/trigger-new/src/lib.rs | 280 +++++++++++++++++++++++++++++++ crates/trigger-new/src/loader.rs | 84 ++++++++++ crates/trigger-new/src/locked.rs | 264 +++++++++++++++++++++++++++++ crates/trigger-new/src/stdio.rs | 60 +++++++ 9 files changed, 929 insertions(+), 3 deletions(-) create mode 100644 crates/trigger-new/Cargo.toml create mode 100644 crates/trigger-new/src/cli.rs create mode 100644 crates/trigger-new/src/lib.rs create mode 100644 crates/trigger-new/src/loader.rs create mode 100644 crates/trigger-new/src/locked.rs create mode 100644 crates/trigger-new/src/stdio.rs diff --git a/Cargo.lock b/Cargo.lock index fab77f082..a6bc5e53e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2969,6 +2969,16 @@ dependencies = [ "regex", ] +[[package]] +name = "sanitize-filename" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c502bdb638f1396509467cb0580ef3b29aa2a45c5d43e5d84928241280296c" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "schannel" version = "0.1.20" @@ -3418,7 +3428,7 @@ dependencies = [ "bytes", "cap-std 0.24.4", "dirs 4.0.0", - "sanitize-filename", + "sanitize-filename 0.3.0", "spin-manifest", "tempfile", "tokio", @@ -3680,6 +3690,35 @@ dependencies = [ "wasmtime", ] +[[package]] +name = "spin-trigger-new" +version = "0.2.0" +dependencies = [ + "anyhow", + "async-trait", + "clap 3.2.19", + "ctrlc", + "dirs 4.0.0", + "futures", + "outbound-http", + "outbound-pg", + "outbound-redis", + "sanitize-filename 0.4.0", + "serde", + "serde_json", + "spin-app", + "spin-config", + "spin-core", + "spin-loader", + "spin-manifest", + "tempfile", + "tokio", + "toml", + "tracing", + "url", + "wasmtime", +] + [[package]] name = "stable_deref_trait" version = "1.2.0" diff --git a/Cargo.toml b/Cargo.toml index afd00c2c8..748b50cec 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -86,6 +86,7 @@ members = [ "crates/templates", "crates/testing", "crates/trigger", + "crates/trigger-new", "examples/spin-timer", "sdk/rust", "sdk/rust/macro" diff --git a/crates/manifest/src/lib.rs b/crates/manifest/src/lib.rs index 000c7974a..4a07ad224 100644 --- a/crates/manifest/src/lib.rs +++ b/crates/manifest/src/lib.rs @@ -2,14 +2,15 @@ #![deny(missing_docs)] -use indexmap::IndexMap; -use serde::{Deserialize, Serialize}; use std::{ collections::HashMap, fmt::{Debug, Formatter}, path::PathBuf, }; +use indexmap::IndexMap; +use serde::{Deserialize, Serialize}; + /// A trigger error. #[derive(Debug, thiserror::Error)] pub enum Error { @@ -222,6 +223,15 @@ impl AllowedHttpHost { } } +impl From for String { + fn from(allowed: AllowedHttpHost) -> Self { + match allowed.port { + Some(port) => format!("{}:{}", allowed.domain, port), + None => allowed.domain, + } + } +} + /// WebAssembly configuration. #[derive(Clone, Debug, Default)] pub struct WasmConfig { diff --git a/crates/trigger-new/Cargo.toml b/crates/trigger-new/Cargo.toml new file mode 100644 index 000000000..3a304ec3f --- /dev/null +++ b/crates/trigger-new/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "spin-trigger-new" +version = "0.2.0" +edition = "2021" +authors = ["Fermyon Engineering "] + +[dependencies] +anyhow = "1.0" +async-trait = "0.1" +clap = { version = "3.1.15", features = ["derive", "env"] } +ctrlc = { version = "3.2", features = ["termination"] } +dirs = "4" +futures = "0.3" +outbound-http = { path = "../outbound-http" } +outbound-redis = { path = "../outbound-redis" } +outbound-pg = { path = "../outbound-pg" } +sanitize-filename = "0.4" +serde = "1.0" +serde_json = "1.0" +spin-app = { path = "../app" } +spin-config = { path = "../config" } +spin-core = { path = "../core" } +spin-loader = { path = "../loader" } +spin-manifest = { path = "../manifest" } +tracing = { version = "0.1", features = [ "log" ] } +url = "2" +wasmtime = "0.39.1" + +[dev-dependencies] +tempfile = "3.3.0" +toml = "0.5" +tokio = { version = "1.0", features = ["rt", "macros"] } \ No newline at end of file diff --git a/crates/trigger-new/src/cli.rs b/crates/trigger-new/src/cli.rs new file mode 100644 index 000000000..51db7aad1 --- /dev/null +++ b/crates/trigger-new/src/cli.rs @@ -0,0 +1,156 @@ +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::{Args, IntoApp, Parser}; +use serde::de::DeserializeOwned; + +use crate::{loader::TriggerLoader, stdio::FollowComponents}; +use crate::{TriggerExecutor, TriggerExecutorBuilder}; + +pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; +pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; +pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; +pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; + +// Set by `spin up` +pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; +pub const SPIN_WORKING_DIR: &str = "SPIN_WORKING_DIR"; + +/// A command that runs a TriggerExecutor. +#[derive(Parser, Debug)] +#[clap(next_help_heading = "TRIGGER OPTIONS")] +pub struct TriggerExecutorCommand +where + Executor::RunConfig: Args, +{ + /// Log directory for the stdout and stderr of components. + #[clap( + name = APP_LOG_DIR, + short = 'L', + long = "log-dir", + )] + pub log: Option, + + /// Disable Wasmtime cache. + #[clap( + name = DISABLE_WASMTIME_CACHE, + long = "disable-cache", + env = DISABLE_WASMTIME_CACHE, + conflicts_with = WASMTIME_CACHE_FILE, + takes_value = false, + )] + pub disable_cache: bool, + + /// Wasmtime cache configuration file. + #[clap( + name = WASMTIME_CACHE_FILE, + long = "cache", + env = WASMTIME_CACHE_FILE, + conflicts_with = DISABLE_WASMTIME_CACHE, + )] + pub cache: Option, + + /// Print output for given component(s) to stdout/stderr + #[clap( + name = FOLLOW_LOG_OPT, + long = "follow", + multiple_occurrences = true, + )] + pub follow_components: Vec, + + /// Print all component output to stdout/stderr + #[clap( + long = "follow-all", + conflicts_with = FOLLOW_LOG_OPT, + )] + pub follow_all_components: bool, + + /// Set the static assets of the components in the temporary directory as writable. + #[clap(long = "allow-transient-write")] + pub allow_transient_write: bool, + + #[clap(flatten)] + pub run_config: Executor::RunConfig, + + #[clap(long = "help-args-only", hide = true)] + pub help_args_only: bool, +} + +/// An empty implementation of clap::Args to be used as TriggerExecutor::RunConfig +/// for executors that do not need additional CLI args. +#[derive(Args)] +pub struct NoArgs; + +impl TriggerExecutorCommand +where + Executor::RunConfig: Args, + Executor::TriggerConfig: DeserializeOwned, +{ + /// Create a new TriggerExecutorBuilder from this TriggerExecutorCommand. + pub async fn run(self) -> Result<()> { + if self.help_args_only { + Self::command() + .disable_help_flag(true) + .help_template("{all-args}") + .print_long_help()?; + return Ok(()); + } + + // Required env vars + let working_dir = std::env::var(SPIN_WORKING_DIR).context(SPIN_WORKING_DIR)?; + let locked_url = std::env::var(SPIN_LOCKED_URL).context(SPIN_LOCKED_URL)?; + + let loader = TriggerLoader::new(working_dir, self.allow_transient_write); + + let executor: Executor = { + let mut builder = TriggerExecutorBuilder::new(loader); + self.update_wasmtime_config(builder.wasmtime_config_mut())?; + builder.follow_components(self.follow_components()); + if let Some(log_dir) = self.log { + builder.log_dir(log_dir); + } + builder.build(locked_url).await? + }; + + let run_fut = executor.run(self.run_config); + + let (abortable, abort_handle) = futures::future::abortable(run_fut); + ctrlc::set_handler(move || abort_handle.abort())?; + match abortable.await { + Ok(Ok(())) => { + tracing::info!("Trigger executor shut down: exiting"); + Ok(()) + } + Ok(Err(err)) => { + tracing::error!("Trigger executor failed: {:?}", err); + Err(err) + } + Err(_aborted) => { + tracing::info!("User requested shutdown: exiting"); + Ok(()) + } + } + } + + pub fn follow_components(&self) -> FollowComponents { + if self.follow_all_components { + FollowComponents::All + } else if self.follow_components.is_empty() { + FollowComponents::None + } else { + let followed = self.follow_components.clone().into_iter().collect(); + FollowComponents::Named(followed) + } + } + + fn update_wasmtime_config(&self, config: &mut spin_core::wasmtime::Config) -> Result<()> { + // Apply --cache / --disable-cache + if !self.disable_cache { + match &self.cache { + Some(p) => config.cache_config_load(p)?, + None => config.cache_config_load_default()?, + }; + } + Ok(()) + } +} diff --git a/crates/trigger-new/src/lib.rs b/crates/trigger-new/src/lib.rs new file mode 100644 index 000000000..3cb7ba2b2 --- /dev/null +++ b/crates/trigger-new/src/lib.rs @@ -0,0 +1,280 @@ +pub mod cli; +mod loader; +pub mod locked; +mod stdio; + +use std::{collections::HashMap, marker::PhantomData, path::PathBuf}; + +use anyhow::{Context, Result}; +use async_trait::async_trait; +use serde::de::DeserializeOwned; + +use spin_app::{App, AppLoader, AppTrigger, Loader, OwnedApp}; +use spin_config::{provider::env::EnvProvider, Provider}; +use spin_core::{Config, Engine, EngineBuilder, Instance, InstancePre, Store, StoreBuilder}; + +use stdio::{ComponentStdioWriter, FollowComponents}; + +const SPIN_HOME: &str = ".spin"; +const SPIN_CONFIG_ENV_PREFIX: &str = "SPIN_APP"; + +#[async_trait] +pub trait TriggerExecutor: Sized { + const TRIGGER_TYPE: &'static str; + type RuntimeData: Default + Send + Sync + 'static; + type TriggerConfig; + type RunConfig; + + /// Create a new trigger executor. + fn new(app_engine: TriggerAppEngine) -> Result; + + /// Run the trigger executor. + async fn run(self, config: Self::RunConfig) -> Result<()>; + + /// Make changes to the ExecutionContext using the given Builder. + fn configure_engine(_builder: &mut EngineBuilder) -> Result<()> { + Ok(()) + } +} + +pub struct TriggerExecutorBuilder { + loader: AppLoader, + config: Config, + log_dir: Option, + follow_components: FollowComponents, + disable_default_host_components: bool, + _phantom: PhantomData, +} + +impl TriggerExecutorBuilder { + /// Create a new TriggerExecutorBuilder with the given Application. + pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { + Self { + loader: AppLoader::new(loader), + config: Default::default(), + log_dir: None, + follow_components: Default::default(), + disable_default_host_components: false, + _phantom: PhantomData, + } + } + + /// !!!Warning!!! Using a custom Wasmtime Config is entirely unsupported; + /// many configurations are likely to cause errors or unexpected behavior. + #[doc(hidden)] + pub fn wasmtime_config_mut(&mut self) -> &mut spin_core::wasmtime::Config { + self.config.wasmtime_config() + } + + pub fn log_dir(&mut self, log_dir: PathBuf) -> &mut Self { + self.log_dir = Some(log_dir); + self + } + + pub fn follow_components(&mut self, follow_components: FollowComponents) -> &mut Self { + self.follow_components = follow_components; + self + } + + pub fn disable_default_host_components(&mut self) -> &mut Self { + self.disable_default_host_components = true; + self + } + + pub async fn build(self, app_uri: String) -> Result + where + Executor::TriggerConfig: DeserializeOwned, + { + let engine = { + let mut builder = Engine::builder(&self.config)?; + + if !self.disable_default_host_components { + // FIXME(lann): migrate host components from prototype + // builder.add_host_component(outbound_redis::OutboundRedis::default())?; + // builder.add_host_component(outbound_pg::OutboundPg)?; + // self.loader.add_dynamic_host_component( + // &mut builder, + // outbound_http::OutboundHttpComponent, + // )?; + // self.loader.add_dynamic_host_component( + // &mut builder, + // spin_config::ConfigHostComponent::new(self.default_config_providers(&app_uri)), + // )?; + } + + Executor::configure_engine(&mut builder)?; + builder.build() + }; + + let app = self.loader.load_owned_app(app_uri).await?; + let app_name = app.require_metadata("name")?; + + let log_dir = { + let sanitized_app = sanitize_filename::sanitize(&app_name); + let parent_dir = match dirs::home_dir() { + Some(home) => home.join(SPIN_HOME), + None => PathBuf::new(), // "./" + }; + parent_dir.join(sanitized_app).join("logs") + }; + std::fs::create_dir_all(&log_dir)?; + + // Run trigger executor + Executor::new( + TriggerAppEngine::new(engine, app_name, app, log_dir, self.follow_components).await?, + ) + } + + pub fn default_config_providers(&self, _app_uri: &str) -> Vec> { + // EnvProvider + // FIXME(lann): Update EnvProvider from prototype + // let dotenv_path = app_uri + // .strip_prefix("file://") + // .and_then(|path| Path::new(path).parent()) + // .unwrap_or_else(|| Path::new(".")) + // .join(".env"); + vec![Box::new(EnvProvider::new( + SPIN_CONFIG_ENV_PREFIX, + Default::default(), + ))] + } +} + +/// Execution context for a TriggerExecutor executing a particular App. +pub struct TriggerAppEngine { + /// Engine to be used with this executor. + pub engine: Engine, + /// Name of the app for e.g. logging. + pub app_name: String, + // An owned wrapper of the App. + app: OwnedApp, + // Log directory + log_dir: PathBuf, + // Component stdio follow config + follow_components: FollowComponents, + // Trigger configs for this trigger type, with order matching `app.triggers_with_type(Executor::TRIGGER_TYPE)` + trigger_configs: Vec, + // Map of {Component ID -> InstancePre} for each component. + component_instance_pres: HashMap>, +} + +impl TriggerAppEngine { + /// Returns a new TriggerAppEngine. May return an error if trigger config validation or + /// component pre-instantiation fails. + pub async fn new( + engine: Engine, + app_name: String, + app: OwnedApp, + log_dir: PathBuf, + follow_components: FollowComponents, + ) -> Result + where + ::TriggerConfig: DeserializeOwned, + { + let trigger_configs = app + .triggers_with_type(Executor::TRIGGER_TYPE) + .map(|trigger| { + trigger.typed_config().with_context(|| { + format!("invalid trigger configuration for {:?}", trigger.id()) + }) + }) + .collect::>()?; + + let mut component_instance_pres = HashMap::default(); + for component in app.components() { + let module = component.load_module(&engine).await?; + let instance_pre = engine.instantiate_pre(&module)?; + component_instance_pres.insert(component.id().to_string(), instance_pre); + } + + Ok(Self { + engine, + app_name, + app, + log_dir, + follow_components, + trigger_configs, + component_instance_pres, + }) + } + + /// Returns a reference to the App. + pub fn app(&self) -> &App { + &self.app + } + + /// Returns AppTriggers and typed TriggerConfigs for this executor type. + pub fn trigger_configs(&self) -> impl Iterator { + self.app + .triggers_with_type(Executor::TRIGGER_TYPE) + .zip(&self.trigger_configs) + } + + /// Returns a new StoreBuilder for the given component ID. + pub fn store_builder(&self, component_id: &str) -> Result { + let mut builder = self.engine.store_builder(); + + // Set up stdio logging + builder.stdout_pipe(self.component_stdio_writer(component_id, "stdout")?); + builder.stderr_pipe(self.component_stdio_writer(component_id, "stderr")?); + + Ok(builder) + } + + fn component_stdio_writer( + &self, + component_id: &str, + log_suffix: &str, + ) -> Result { + let sanitized_component_id = sanitize_filename::sanitize(component_id); + // e.g. + let log_path = self + .log_dir + .join(format!("{sanitized_component_id}_{log_suffix}.txt")); + let follow = self.follow_components.should_follow(component_id); + ComponentStdioWriter::new(&log_path, follow) + .with_context(|| format!("Failed to open log file {log_path:?}")) + } + + /// Returns a new Store and Instance for the given component ID. + pub async fn prepare_instance( + &self, + component_id: &str, + ) -> Result<(Instance, Store)> { + let store_builder = self.store_builder(component_id)?; + self.prepare_instance_with_store(component_id, store_builder) + .await + } + + /// Returns a new Store and Instance for the given component ID and StoreBuilder. + pub async fn prepare_instance_with_store( + &self, + component_id: &str, + mut store_builder: StoreBuilder, + ) -> Result<(Instance, Store)> { + // Look up AppComponent + let component = self.app.get_component(component_id).with_context(|| { + format!( + "app {:?} has no component {:?}", + self.app_name, component_id + ) + })?; + + // Build Store + component.apply_store_config(&mut store_builder).await?; + let mut store = store_builder.build()?; + + // Instantiate + let instance = self.component_instance_pres[component_id] + .instantiate_async(&mut store) + .await + .with_context(|| { + format!( + "app {:?} component {:?} instantiation failed", + self.app_name, component_id + ) + })?; + + Ok((instance, store)) + } +} diff --git a/crates/trigger-new/src/loader.rs b/crates/trigger-new/src/loader.rs new file mode 100644 index 000000000..9ae915290 --- /dev/null +++ b/crates/trigger-new/src/loader.rs @@ -0,0 +1,84 @@ +#![allow(dead_code)] // Refactor WIP + +use std::path::{Path, PathBuf}; + +use anyhow::{ensure, Context, Result}; +use async_trait::async_trait; +use spin_app::{ + locked::{LockedApp, LockedComponentSource}, + AppComponent, Loader, +}; +use spin_core::StoreBuilder; + +pub struct TriggerLoader { + working_dir: PathBuf, + allow_transient_write: bool, +} + +impl TriggerLoader { + pub(crate) fn new(working_dir: impl Into, allow_transient_write: bool) -> Self { + Self { + working_dir: working_dir.into(), + allow_transient_write, + } + } +} + +#[async_trait] +impl Loader for TriggerLoader { + async fn load_app(&self, uri: &str) -> Result { + let path = unwrap_file_uri(uri)?; + let contents = + std::fs::read(path).with_context(|| format!("failed to read manifest at {path:?}"))?; + let app = + serde_json::from_slice(&contents).context("failed to parse app lock file JSON")?; + Ok(app) + } + + async fn load_module( + &self, + engine: &spin_core::wasmtime::Engine, + source: &LockedComponentSource, + ) -> Result { + let source = source + .content + .source + .as_ref() + .context("LockedComponentSource missing source field")?; + let path = unwrap_file_uri(source)?; + spin_core::Module::from_file(engine, path) + } + + async fn mount_files( + &self, + store_builder: &mut StoreBuilder, + component: &AppComponent, + ) -> Result<()> { + for content_dir in component.files() { + let source_uri = content_dir + .content + .source + .as_deref() + .with_context(|| format!("Missing 'source' on files mount {content_dir:?}"))?; + let source_path = self.working_dir.join(unwrap_file_uri(source_uri)?); + ensure!( + source_path.is_dir(), + "TriggerLoader only supports directory mounts; {source_path:?} is not a directory" + ); + let guest_path = content_dir.path.clone(); + if self.allow_transient_write { + store_builder.read_write_preopened_dir(source_path, guest_path)?; + } else { + store_builder.read_only_preopened_dir(source_path, guest_path)?; + } + } + Ok(()) + } +} + +fn unwrap_file_uri(uri: &str) -> Result<&Path> { + Ok(Path::new( + uri.strip_prefix("file://") + .context("TriggerLoader supports only file:// URIs")?, + )) +} diff --git a/crates/trigger-new/src/locked.rs b/crates/trigger-new/src/locked.rs new file mode 100644 index 000000000..cd81e0b95 --- /dev/null +++ b/crates/trigger-new/src/locked.rs @@ -0,0 +1,264 @@ +#![allow(dead_code)] // Refactor WIP + +use std::path::{Path, PathBuf}; + +use anyhow::{anyhow, bail, Context, Result}; +use outbound_http::ALLOW_ALL_HOSTS; +use spin_app::{ + locked::{ + self, ContentPath, ContentRef, LockedApp, LockedComponent, LockedComponentSource, + LockedTrigger, + }, + values::{ValuesMap, ValuesMapBuilder}, +}; +use spin_manifest::{ + AllowedHttpHosts, Application, ApplicationInformation, ApplicationTrigger, CoreComponent, + HttpConfig, HttpTriggerConfiguration, RedisConfig, TriggerConfig, +}; + +const WASM_CONTENT_TYPE: &str = "application/wasm"; + +/// Construct a LockedApp from the given Application. Any buffered component +/// sources will be written to the given `working_dir`. +pub fn build_locked_app(app: Application, working_dir: impl Into) -> Result { + let working_dir = working_dir.into(); + LockedAppBuilder { working_dir }.build(app) +} + +struct LockedAppBuilder { + working_dir: PathBuf, +} + +// TODO(lann): Consolidate metadata w/ spin-manifest models +impl LockedAppBuilder { + fn build(self, app: Application) -> Result { + Ok(LockedApp { + spin_lock_version: spin_app::locked::FixedVersion, + triggers: self.build_triggers(&app.info.trigger, app.component_triggers)?, + metadata: self.build_metadata(app.info)?, + variables: self.build_variables(app.variables)?, + components: self.build_components(app.components)?, + }) + } + + fn build_metadata(&self, info: ApplicationInformation) -> Result { + let mut builder = ValuesMapBuilder::new(); + builder + .string("name", &info.name) + .string("version", &info.version) + .string_option("description", info.description.as_deref()) + .serializable("trigger", info.trigger)?; + // Convert ApplicationOrigin to a URI + builder.string( + "origin", + match info.origin { + spin_manifest::ApplicationOrigin::File(path) => file_uri(&path, false)?, + spin_manifest::ApplicationOrigin::Bindle { id, server } => { + format!("bindle+{server}?id={id}") + } + }, + ); + Ok(builder.build()) + } + + fn build_variables>( + &self, + variables: impl IntoIterator, + ) -> Result { + variables + .into_iter() + .map(|(name, var)| { + Ok(( + name, + locked::Variable { + default: var.default, + secret: var.secret, + }, + )) + }) + .collect() + } + + fn build_triggers( + &self, + app_trigger: &ApplicationTrigger, + component_triggers: impl IntoIterator, + ) -> Result> { + component_triggers + .into_iter() + .map(|(component_id, config)| { + let id = format!("trigger--{component_id}"); + let mut builder = ValuesMapBuilder::new(); + builder.string("component", component_id); + + let trigger_type; + match (app_trigger, config) { + (ApplicationTrigger::Http(HttpTriggerConfiguration{base}), TriggerConfig::Http(HttpConfig{ route, executor })) => { + trigger_type = "http"; + let route = base.trim_end_matches('/').to_string() + &route; + builder.string("route", route); + builder.serializable("executor", executor)?; + }, + (ApplicationTrigger::Redis(_), TriggerConfig::Redis(RedisConfig{ channel, executor: _ })) => { + trigger_type = "redis"; + builder.string("channel", channel); + }, + (app_config, trigger_config) => bail!("Mismatched app and component trigger configs: {app_config:?} vs {trigger_config:?}") + } + + Ok(LockedTrigger { + id, + trigger_type: trigger_type.into(), + trigger_config: builder.build().into() + }) + }) + .collect() + } + + fn build_components( + &self, + components: impl IntoIterator, + ) -> Result> { + components + .into_iter() + .map(|component| self.build_component(component)) + .collect() + } + + fn build_component(&self, component: CoreComponent) -> Result { + let id = component.id; + + let metadata = ValuesMapBuilder::new() + .string_option("description", component.description) + .string_array( + "allowed_http_hosts", + allowed_http_hosts_to_strings(component.wasm.allowed_http_hosts), + ) + .build(); + + let source = { + let path = match component.source { + spin_manifest::ModuleSource::FileReference(path) => path, + spin_manifest::ModuleSource::Buffer(bytes, name) => { + let wasm_path = self.working_dir.join(&id).with_extension("wasm"); + std::fs::write(&wasm_path, bytes).with_context(|| { + format!("Failed to write buffered source for component {id:?} ({name})") + })?; + wasm_path + } + }; + LockedComponentSource { + content_type: WASM_CONTENT_TYPE.into(), + content: content_ref_path(&path, false)?, + } + }; + + let env = component.wasm.environment.into_iter().collect(); + + let files = component + .wasm + .mounts + .into_iter() + .map(|mount| { + Ok(ContentPath { + content: content_ref_path(&mount.host, true)?, + path: mount.guest.into(), + }) + }) + .collect::>()?; + + let config = component.config.into_iter().collect(); + + Ok(LockedComponent { + id, + metadata, + source, + env, + files, + config, + }) + } +} + +fn content_ref_path(path: &Path, is_dir: bool) -> Result { + Ok(ContentRef { + source: Some(file_uri(path, is_dir)?), + ..Default::default() + }) +} + +fn file_uri(path: &Path, is_dir: bool) -> Result { + let uri = if is_dir { + url::Url::from_directory_path(path) + } else { + url::Url::from_file_path(path) + } + .map_err(|err| anyhow!("Could not construct file URI: {err:?}"))?; + Ok(uri.to_string()) +} + +fn allowed_http_hosts_to_strings(allowed_hosts: AllowedHttpHosts) -> Vec { + match allowed_hosts { + AllowedHttpHosts::AllowAll => vec![ALLOW_ALL_HOSTS.into()], + AllowedHttpHosts::AllowSpecific(hosts) => hosts.into_iter().map(Into::into).collect(), + } +} + +#[cfg(test)] +mod tests { + use tempfile::TempDir; + + use super::*; + + const TEST_MANIFEST: &str = r#" + spin_version = "1" + name = "test-app" + version = "0.0.1" + trigger = { type = "http", base = "/" } + + [variables] + test_var = { default = "test-val" } + + [[component]] + id = "test-component" + source = "test-source.wasm" + files = ["content/*"] + allowed_http_hosts = ["example.com"] + [component.config] + test_config = "{{test_var}}" + [component.trigger] + route = "/" + + [[component]] + id = "test-component-2" + source = "test-source-2.wasm" + allowed_http_hosts = ["insecure:allow-all"] + [component.trigger] + route = "/other" + "#; + + async fn test_app() -> (Application, TempDir) { + let tempdir = TempDir::new().expect("tempdir"); + std::env::set_current_dir(tempdir.path()).unwrap(); + std::fs::write("spin.toml", TEST_MANIFEST).expect("write manifest"); + let app = spin_loader::local::from_file("spin.toml", &tempdir, &None, false) + .await + .expect("load app"); + (app, tempdir) + } + + #[tokio::test] + async fn build_locked_app_smoke_test() { + let (app, tempdir) = test_app().await; + let locked = build_locked_app(app, tempdir.path()).unwrap(); + assert_eq!(locked.metadata["name"], "test-app"); + assert!(locked.variables.contains_key("test_var")); + assert_eq!(locked.triggers[0].trigger_config["route"], "/"); + + let component = &locked.components[0]; + let source = component.source.content.source.as_deref().unwrap(); + assert!(source.ends_with("test-source.wasm")); + let mount = component.files[0].content.source.as_deref().unwrap(); + assert!(mount.ends_with('/')); + } +} diff --git a/crates/trigger-new/src/stdio.rs b/crates/trigger-new/src/stdio.rs new file mode 100644 index 000000000..80d688032 --- /dev/null +++ b/crates/trigger-new/src/stdio.rs @@ -0,0 +1,60 @@ +use std::{collections::HashSet, fs::File, path::Path}; + +/// Which components should have their logs followed on stdout/stderr. +#[derive(Clone, Debug)] +pub enum FollowComponents { + /// No components should have their logs followed. + None, + /// Only the specified components should have their logs followed. + Named(HashSet), + /// All components should have their logs followed. + All, +} + +impl FollowComponents { + /// Whether a given component should have its logs followed on stdout/stderr. + pub fn should_follow(&self, component_id: &str) -> bool { + match self { + Self::None => false, + Self::All => true, + Self::Named(ids) => ids.contains(component_id), + } + } +} + +impl Default for FollowComponents { + fn default() -> Self { + Self::None + } +} + +/// ComponentStdioWriter forwards output to a log file and (optionally) stderr +pub struct ComponentStdioWriter { + log_file: File, + follow: bool, +} + +impl ComponentStdioWriter { + pub fn new(log_path: &Path, follow: bool) -> anyhow::Result { + let log_file = File::options().create(true).append(true).open(log_path)?; + Ok(Self { log_file, follow }) + } +} + +impl std::io::Write for ComponentStdioWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + let written = self.log_file.write(buf)?; + if self.follow { + std::io::stderr().write_all(&buf[..written])?; + } + Ok(written) + } + + fn flush(&mut self) -> std::io::Result<()> { + self.log_file.flush()?; + if self.follow { + std::io::stderr().flush()?; + } + Ok(()) + } +} From 2be489fd5bfb79b0eb1d9ff0e7d9d2c2aaebcdb7 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 17:08:16 -0400 Subject: [PATCH 04/28] Migrate spin-config to new core Signed-off-by: Lann Martin --- Cargo.lock | 7 +-- crates/config/Cargo.toml | 8 +-- crates/config/src/host_component.rs | 83 ++++++++++++++++++++--------- crates/config/src/lib.rs | 28 +++++----- crates/config/src/provider/env.rs | 77 ++++++++++++++++---------- crates/trigger-new/src/lib.rs | 31 ++++++----- crates/trigger/Cargo.toml | 1 - crates/trigger/src/lib.rs | 51 +----------------- 8 files changed, 149 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a6bc5e53e..0355d4a1b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3399,8 +3399,10 @@ version = "0.2.0" dependencies = [ "anyhow", "async-trait", - "spin-engine", - "spin-manifest", + "dotenvy", + "once_cell", + "spin-app", + "spin-core", "thiserror", "tokio", "toml", @@ -3682,7 +3684,6 @@ dependencies = [ "outbound-http", "outbound-pg", "outbound-redis", - "spin-config", "spin-engine", "spin-loader", "spin-manifest", diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index fe82cefd3..960dbe38a 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -7,9 +7,12 @@ authors = [ "Fermyon Engineering " ] [dependencies] anyhow = "1.0" async-trait = "0.1" -spin-engine = { path = "../engine" } -spin-manifest = { path = "../manifest" } +dotenvy = "0.15" +once_cell = "1" +spin-app = { path = "../app" } +spin-core = { path = "../core" } thiserror = "1" +tokio = { version = "1", features = ["rt-multi-thread"] } [dependencies.wit-bindgen-wasmtime] git = "https://github.com/bytecodealliance/wit-bindgen" @@ -17,5 +20,4 @@ rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" features = ["async"] [dev-dependencies] -tokio = { version = "1", features = [ "rt-multi-thread" ] } toml = "0.5" diff --git a/crates/config/src/host_component.rs b/crates/config/src/host_component.rs index 1ac0a94e7..bfa1a88c3 100644 --- a/crates/config/src/host_component.rs +++ b/crates/config/src/host_component.rs @@ -1,60 +1,93 @@ -use std::sync::Arc; +use std::sync::{Arc, Mutex}; -use spin_engine::host_component::HostComponent; -use spin_manifest::CoreComponent; -use wit_bindgen_wasmtime::async_trait; +use async_trait::async_trait; +use once_cell::sync::OnceCell; +use spin_app::{AppComponent, DynamicHostComponent}; +use spin_core::HostComponent; -use crate::{Error, Key, Resolver}; +use crate::{Error, Key, Provider, Resolver}; -mod wit { - wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/spin-config.wit"], async: *}); -} +wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/spin-config.wit"], async: *}); pub struct ConfigHostComponent { - resolver: Arc, + providers: Mutex>>, + resolver: Arc>, } impl ConfigHostComponent { - pub fn new(resolver: Resolver) -> Self { + pub fn new(providers: Vec>) -> Self { Self { - resolver: Arc::new(resolver), + providers: Mutex::new(providers), + resolver: Default::default(), } } } impl HostComponent for ConfigHostComponent { - type State = ComponentConfig; + type Data = ComponentConfig; fn add_to_linker( - linker: &mut wit_bindgen_wasmtime::wasmtime::Linker>, - state_handle: spin_engine::host_component::HostComponentsStateHandle, + linker: &mut spin_core::Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, ) -> anyhow::Result<()> { - wit::spin_config::add_to_linker(linker, move |ctx| state_handle.get_mut(ctx)) + spin_config::add_to_linker(linker, get) } - fn build_state(&self, component: &CoreComponent) -> anyhow::Result { - Ok(ComponentConfig { - component_id: component.id.clone(), + fn build_data(&self) -> Self::Data { + ComponentConfig { resolver: self.resolver.clone(), - }) + component_id: None, + } + } +} + +impl DynamicHostComponent for ConfigHostComponent { + fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()> { + self.resolver.get_or_try_init(|| { + let mut resolver = Resolver::new( + component + .app + .variables() + .map(|(key, var)| (key.clone(), var.clone())), + )?; + for component in component.app.components() { + resolver.add_component_config( + component.id(), + component.config().map(|(k, v)| (k.into(), v.into())), + )?; + } + for provider in self.providers.lock().unwrap().drain(..) { + resolver.add_provider(provider); + } + Ok::<_, anyhow::Error>(resolver) + })?; + data.component_id = Some(component.id().to_string()); + Ok(()) } } /// A component configuration interface implementation. pub struct ComponentConfig { - component_id: String, - resolver: Arc, + resolver: Arc>, + component_id: Option, } #[async_trait] -impl wit::spin_config::SpinConfig for ComponentConfig { - async fn get_config(&mut self, key: &str) -> Result { +impl spin_config::SpinConfig for ComponentConfig { + async fn get_config(&mut self, key: &str) -> Result { + // Set by DynamicHostComponent::update_data + let component_id = self.component_id.as_deref().unwrap(); let key = Key::new(key)?; - Ok(self.resolver.resolve(&self.component_id, key).await?) + Ok(self + .resolver + .get() + .unwrap() + .resolve(component_id, key) + .await?) } } -impl From for wit::spin_config::Error { +impl From for spin_config::Error { fn from(err: Error) -> Self { match err { Error::InvalidKey(msg) => Self::InvalidKey(msg), diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 16a4faa54..28573e7b0 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -1,25 +1,21 @@ -pub mod host_component; +mod host_component; pub mod provider; - mod template; use std::{borrow::Cow, collections::HashMap, fmt::Debug}; -pub use async_trait::async_trait; +use spin_app::Variable; -pub use provider::Provider; -use spin_manifest::Variable; +pub use crate::{host_component::ConfigHostComponent, provider::Provider}; use template::{Part, Template}; -type Result = std::result::Result; - /// A configuration resolver. #[derive(Debug, Default)] pub struct Resolver { // variable key -> variable variables: HashMap, // component ID -> config key -> config value template - components_configs: HashMap>, + component_configs: HashMap>, providers: Vec>, } @@ -31,7 +27,7 @@ impl Resolver { variables.keys().try_for_each(|key| Key::validate(key))?; Ok(Self { variables, - components_configs: Default::default(), + component_configs: Default::default(), providers: Default::default(), }) } @@ -53,19 +49,19 @@ impl Resolver { }) .collect::>()?; - self.components_configs.insert(component_id, templates); + self.component_configs.insert(component_id, templates); Ok(()) } /// Adds a config Provider to the Resolver. - pub fn add_provider(&mut self, provider: impl Provider + 'static) { - self.providers.push(Box::new(provider)); + pub fn add_provider(&mut self, provider: Box) { + self.providers.push(provider); } /// Resolves a config value for the given path. pub async fn resolve(&self, component_id: &str, key: Key<'_>) -> Result { - let configs = self.components_configs.get(component_id).ok_or_else(|| { + let configs = self.component_configs.get(component_id).ok_or_else(|| { Error::UnknownPath(format!("no config for component {component_id:?}")) })?; @@ -165,6 +161,8 @@ impl<'a> AsRef for Key<'a> { } } +type Result = std::result::Result; + /// A config resolution error. #[derive(Debug, thiserror::Error)] pub enum Error { @@ -195,6 +193,8 @@ pub enum Error { #[cfg(test)] mod tests { + use async_trait::async_trait; + use super::*; #[derive(Debug)] @@ -235,7 +235,7 @@ mod tests { [("test_key".into(), config_template.into())], ) .unwrap(); - resolver.add_provider(TestProvider); + resolver.add_provider(Box::new(TestProvider)); resolver.resolve("test-component", Key("test_key")).await } diff --git a/crates/config/src/provider/env.rs b/crates/config/src/provider/env.rs index ed2dbfc2f..354fa84fc 100644 --- a/crates/config/src/provider/env.rs +++ b/crates/config/src/provider/env.rs @@ -1,59 +1,73 @@ -use std::collections::HashMap; +use std::{collections::HashMap, path::PathBuf, sync::Mutex}; -use anyhow::Context; +use anyhow::{Context, Result}; use async_trait::async_trait; use crate::{Key, Provider}; -pub const DEFAULT_PREFIX: &str = "SPIN_APP"; - /// A config Provider that uses environment variables. #[derive(Debug)] pub struct EnvProvider { prefix: String, - envs: HashMap, + dotenv_path: Option, + dotenv_cache: Mutex>>, } impl EnvProvider { /// Creates a new EnvProvider. - pub fn new(prefix: impl Into, envs: HashMap) -> Self { + pub fn new(prefix: impl Into, dotenv_path: Option) -> Self { Self { prefix: prefix.into(), - envs, + dotenv_path, + dotenv_cache: Default::default(), } } - fn get_sync(&self, key: &Key) -> anyhow::Result> { + fn get_sync(&self, key: &Key) -> Result> { let env_key = format!("{}_{}", &self.prefix, key.as_ref().to_ascii_uppercase()); match std::env::var(&env_key) { - Err(std::env::VarError::NotPresent) => { - Ok(self.envs.get(&env_key).map(|value| value.to_string())) - } + Err(std::env::VarError::NotPresent) => self.get_dotenv(&env_key), other => other .map(Some) .with_context(|| format!("failed to resolve env var {}", &env_key)), } } -} -impl Default for EnvProvider { - fn default() -> Self { - Self { - prefix: DEFAULT_PREFIX.to_string(), - envs: HashMap::new(), + fn get_dotenv(&self, key: &str) -> Result> { + if self.dotenv_path.is_none() { + return Ok(None); } + let mut maybe_cache = self + .dotenv_cache + .lock() + .expect("dotenv_cache lock poisoned"); + let cache = match maybe_cache.as_mut() { + Some(cache) => cache, + None => maybe_cache.insert(self.load_dotenv()?), + }; + Ok(cache.get(key).cloned()) + } + + fn load_dotenv(&self) -> Result> { + let path = self.dotenv_path.as_deref().unwrap(); + Ok(dotenvy::from_path_iter(path) + .into_iter() + .flatten() + .collect::, _>>()?) } } #[async_trait] impl Provider for EnvProvider { - async fn get(&self, key: &Key) -> anyhow::Result> { - self.get_sync(key) + async fn get(&self, key: &Key) -> Result> { + tokio::task::block_in_place(|| self.get_sync(key)) } } #[cfg(test)] mod test { + use std::env::temp_dir; + use super::*; #[test] @@ -66,20 +80,22 @@ mod test { "dotenv_val".to_string(), ); assert_eq!( - EnvProvider::new("TESTING_SPIN", envs.clone()) + EnvProvider::new("TESTING_SPIN", None) .get_sync(&key1) .unwrap(), Some("val".to_string()) ); + } - let key2 = Key::new("env_key2").unwrap(); - envs.insert( - "TESTING_SPIN_ENV_KEY2".to_string(), - "dotenv_val".to_string(), - ); + #[test] + fn provider_get_dotenv() { + let dotenv_path = temp_dir().join("spin-env-provider-test"); + std::fs::write(&dotenv_path, b"TESTING_SPIN_ENV_KEY2=dotenv_val").unwrap(); + + let key = Key::new("env_key2").unwrap(); assert_eq!( - EnvProvider::new("TESTING_SPIN", envs.clone()) - .get_sync(&key2) + EnvProvider::new("TESTING_SPIN", Some(dotenv_path)) + .get_sync(&key) .unwrap(), Some("dotenv_val".to_string()) ); @@ -88,6 +104,11 @@ mod test { #[test] fn provider_get_missing() { let key = Key::new("please_do_not_ever_set_this_during_tests").unwrap(); - assert_eq!(EnvProvider::default().get_sync(&key).unwrap(), None); + assert_eq!( + EnvProvider::new("TESTING_SPIN", Default::default()) + .get_sync(&key) + .unwrap(), + None + ); } } diff --git a/crates/trigger-new/src/lib.rs b/crates/trigger-new/src/lib.rs index 3cb7ba2b2..a229cf786 100644 --- a/crates/trigger-new/src/lib.rs +++ b/crates/trigger-new/src/lib.rs @@ -3,7 +3,11 @@ mod loader; pub mod locked; mod stdio; -use std::{collections::HashMap, marker::PhantomData, path::PathBuf}; +use std::{ + collections::HashMap, + marker::PhantomData, + path::{Path, PathBuf}, +}; use anyhow::{Context, Result}; use async_trait::async_trait; @@ -81,7 +85,7 @@ impl TriggerExecutorBuilder { self } - pub async fn build(self, app_uri: String) -> Result + pub async fn build(mut self, app_uri: String) -> Result where Executor::TriggerConfig: DeserializeOwned, { @@ -96,10 +100,10 @@ impl TriggerExecutorBuilder { // &mut builder, // outbound_http::OutboundHttpComponent, // )?; - // self.loader.add_dynamic_host_component( - // &mut builder, - // spin_config::ConfigHostComponent::new(self.default_config_providers(&app_uri)), - // )?; + self.loader.add_dynamic_host_component( + &mut builder, + spin_config::ConfigHostComponent::new(self.default_config_providers(&app_uri)), + )?; } Executor::configure_engine(&mut builder)?; @@ -125,17 +129,16 @@ impl TriggerExecutorBuilder { ) } - pub fn default_config_providers(&self, _app_uri: &str) -> Vec> { + pub fn default_config_providers(&self, app_uri: &str) -> Vec> { // EnvProvider - // FIXME(lann): Update EnvProvider from prototype - // let dotenv_path = app_uri - // .strip_prefix("file://") - // .and_then(|path| Path::new(path).parent()) - // .unwrap_or_else(|| Path::new(".")) - // .join(".env"); + let dotenv_path = app_uri + .strip_prefix("file://") + .and_then(|path| Path::new(path).parent()) + .unwrap_or_else(|| Path::new(".")) + .join(".env"); vec![Box::new(EnvProvider::new( SPIN_CONFIG_ENV_PREFIX, - Default::default(), + Some(dotenv_path), ))] } } diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml index d75af9c68..fdbadc7c6 100644 --- a/crates/trigger/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -14,7 +14,6 @@ futures = "0.3" outbound-http = { path = "../outbound-http" } outbound-redis = { path = "../outbound-redis" } outbound-pg = { path = "../outbound-pg" } -spin-config = { path = "../config" } spin-engine = { path = "../engine" } spin-loader = { path = "../loader" } spin-manifest = { path = "../manifest" } diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 1cf6186e6..3a04b6e47 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -2,17 +2,16 @@ use std::{ collections::HashMap, error::Error, marker::PhantomData, - path::{Path, PathBuf}, + path::PathBuf, sync::{Arc, RwLock}, }; use anyhow::Result; use async_trait::async_trait; -use spin_config::{host_component::ConfigHostComponent, Resolver}; use spin_engine::{ io::FollowComponents, Builder, Engine, ExecutionContext, ExecutionContextConfiguration, }; -use spin_manifest::{Application, ApplicationOrigin, ApplicationTrigger, TriggerConfig, Variable}; +use spin_manifest::{Application, ApplicationTrigger, TriggerConfig}; pub mod cli; #[async_trait] @@ -93,12 +92,6 @@ impl TriggerExecutorBuilder { { let app = self.application; - // The .env file is either a sibling to the manifest file or (for bindles) in the current dir. - let dotenv_root = match &app.info.origin { - ApplicationOrigin::File(path) => path.parent().unwrap(), - ApplicationOrigin::Bindle { .. } => Path::new("."), - }; - // Build ExecutionContext let ctx_config = ExecutionContextConfiguration { components: app.components, @@ -111,7 +104,6 @@ impl TriggerExecutorBuilder { ctx_builder.link_defaults()?; if !self.disable_default_host_components { add_default_host_components(&mut ctx_builder)?; - add_config_host_component(&mut ctx_builder, app.variables, dotenv_root)?; } Executor::configure_execution_context(&mut ctx_builder)?; @@ -143,42 +135,3 @@ pub fn add_default_host_components( })?; Ok(()) } - -pub fn add_config_host_component( - ctx_builder: &mut Builder, - variables: HashMap, - dotenv_path: &Path, -) -> Result<()> { - let mut resolver = Resolver::new(variables)?; - - // Add all component configs to the Resolver. - for component in &ctx_builder.config().components { - resolver.add_component_config( - &component.id, - component.config.iter().map(|(k, v)| (k.clone(), v.clone())), - )?; - } - - let envs = read_dotenv(dotenv_path)?; - - // Add default config provider(s). - // TODO(lann): Make config provider(s) configurable. - resolver.add_provider(spin_config::provider::env::EnvProvider::new( - spin_config::provider::env::DEFAULT_PREFIX, - envs, - )); - - ctx_builder.add_host_component(ConfigHostComponent::new(resolver))?; - Ok(()) -} - -// Return environment key value mapping in ".env" file. -fn read_dotenv(dotenv_root: &Path) -> Result> { - let dotenv_path = dotenv_root.join(".env"); - if !dotenv_path.is_file() { - return Ok(Default::default()); - } - dotenvy::from_path_iter(dotenv_path)? - .map(|item| Ok(item?)) - .collect() -} From ad71a332d275bba652adde5ebbae21462aa4955d Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 17:24:21 -0400 Subject: [PATCH 05/28] Migrate outbound-http to new core Signed-off-by: Lann Martin --- Cargo.lock | 6 +- crates/loader/src/bindle/mod.rs | 21 ++--- crates/loader/src/lib.rs | 1 - crates/loader/src/local/mod.rs | 19 ++-- crates/manifest/src/lib.rs | 75 +--------------- crates/outbound-http/Cargo.toml | 11 ++- .../src/allowed_http_hosts.rs} | 74 +++++++++++++-- crates/outbound-http/src/host_component.rs | 41 +++++---- crates/outbound-http/src/lib.rs | 90 +++++++------------ crates/trigger-new/src/lib.rs | 8 +- crates/trigger-new/src/locked.rs | 17 +--- crates/trigger/src/lib.rs | 1 - 12 files changed, 160 insertions(+), 204 deletions(-) rename crates/{loader/src/validation.rs => outbound-http/src/allowed_http_hosts.rs} (81%) diff --git a/Cargo.lock b/Cargo.lock index 0355d4a1b..1775f6279 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2223,12 +2223,10 @@ name = "outbound-http" version = "0.2.0" dependencies = [ "anyhow", - "bytes", - "futures", "http", "reqwest", - "spin-engine", - "spin-manifest", + "spin-app", + "spin-core", "tracing", "url", "wit-bindgen-wasmtime", diff --git a/crates/loader/src/bindle/mod.rs b/crates/loader/src/bindle/mod.rs index 39842a8a5..8674ab6c7 100644 --- a/crates/loader/src/bindle/mod.rs +++ b/crates/loader/src/bindle/mod.rs @@ -10,23 +10,24 @@ mod connection; /// Bindle helper functions. mod utils; -use crate::{ - bindle::{ - config::{RawAppManifest, RawComponentManifest}, - utils::{find_manifest, parcels_in_group}, - }, - validation::{parse_allowed_http_hosts, validate_allowed_http_hosts}, -}; +use std::path::Path; + use anyhow::{anyhow, Context, Result}; use bindle::Invoice; -pub use connection::BindleConnectionInfo; use futures::future; + +use outbound_http::allowed_http_hosts::validate_allowed_http_hosts; use spin_manifest::{ Application, ApplicationInformation, ApplicationOrigin, CoreComponent, ModuleSource, SpinVersion, WasmConfig, }; -use std::path::Path; use tracing::log; + +use crate::bindle::{ + config::{RawAppManifest, RawComponentManifest}, + utils::{find_manifest, parcels_in_group}, +}; +pub use connection::BindleConnectionInfo; pub(crate) use utils::BindleReader; pub use utils::SPIN_MANIFEST_MEDIA_TYPE; @@ -142,7 +143,7 @@ async fn core( None => vec![], }; let environment = raw.wasm.environment.unwrap_or_default(); - let allowed_http_hosts = parse_allowed_http_hosts(&raw.wasm.allowed_http_hosts)?; + let allowed_http_hosts = raw.wasm.allowed_http_hosts.unwrap_or_default(); let wasm = WasmConfig { environment, mounts, diff --git a/crates/loader/src/lib.rs b/crates/loader/src/lib.rs index 34f4e8263..d9dc0020b 100644 --- a/crates/loader/src/lib.rs +++ b/crates/loader/src/lib.rs @@ -14,7 +14,6 @@ mod assets; pub mod bindle; mod common; pub mod local; -mod validation; /// Load a Spin application configuration from a spin.toml manifest file. pub use local::from_file; diff --git a/crates/loader/src/local/mod.rs b/crates/loader/src/local/mod.rs index 17a7714f1..aedf356dd 100644 --- a/crates/loader/src/local/mod.rs +++ b/crates/loader/src/local/mod.rs @@ -10,21 +10,20 @@ pub mod config; #[cfg(test)] mod tests; +use std::{path::Path, str::FromStr}; + use anyhow::{anyhow, bail, Context, Result}; -use config::{RawAppInformation, RawAppManifest, RawAppManifestAnyVersion, RawComponentManifest}; use futures::future; +use outbound_http::allowed_http_hosts::validate_allowed_http_hosts; use path_absolutize::Absolutize; use spin_manifest::{ Application, ApplicationInformation, ApplicationOrigin, CoreComponent, ModuleSource, SpinVersion, WasmConfig, }; -use std::{path::Path, str::FromStr}; use tokio::{fs::File, io::AsyncReadExt}; -use crate::{ - bindle::BindleConnectionInfo, - validation::{parse_allowed_http_hosts, validate_allowed_http_hosts}, -}; +use crate::bindle::BindleConnectionInfo; +use config::{RawAppInformation, RawAppManifest, RawAppManifestAnyVersion, RawComponentManifest}; /// Given the path to a spin.toml manifest file, prepare its assets locally and /// get a prepared application configuration consumable by a Spin execution context. @@ -103,11 +102,9 @@ fn error_on_duplicate_ids(components: Vec) -> Result<()> { pub fn validate_raw_app_manifest(raw: &RawAppManifestAnyVersion) -> Result<()> { match raw { RawAppManifestAnyVersion::V1(raw) => { - let _ = raw - .components + raw.components .iter() - .map(|c| validate_allowed_http_hosts(&c.wasm.allowed_http_hosts)) - .collect::>>()?; + .try_for_each(|c| validate_allowed_http_hosts(&c.wasm.allowed_http_hosts))?; } } Ok(()) @@ -225,7 +222,7 @@ async fn core( None => vec![], }; let environment = raw.wasm.environment.unwrap_or_default(); - let allowed_http_hosts = parse_allowed_http_hosts(&raw.wasm.allowed_http_hosts)?; + let allowed_http_hosts = raw.wasm.allowed_http_hosts.unwrap_or_default(); let wasm = WasmConfig { environment, mounts, diff --git a/crates/manifest/src/lib.rs b/crates/manifest/src/lib.rs index 4a07ad224..2e8ea0d5d 100644 --- a/crates/manifest/src/lib.rs +++ b/crates/manifest/src/lib.rs @@ -159,79 +159,6 @@ impl TryFrom for RedisTriggerConfiguration { } } -/// An HTTP host allow-list. -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum AllowedHttpHosts { - /// All HTTP hosts are allowed (the "insecure:allow-all" value was present in the list) - AllowAll, - /// Only the specified hosts are allowed. - AllowSpecific(Vec), -} - -impl Default for AllowedHttpHosts { - fn default() -> Self { - Self::AllowSpecific(vec![]) - } -} - -impl AllowedHttpHosts { - /// Tests whether the given URL is allowed according to the allow-list. - pub fn allow(&self, url: &url::Url) -> bool { - match self { - Self::AllowAll => true, - Self::AllowSpecific(hosts) => hosts.iter().any(|h| h.allow(url)), - } - } -} - -/// An HTTP host allow-list entry. -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct AllowedHttpHost { - domain: String, - port: Option, -} - -impl AllowedHttpHost { - /// Creates a new allow-list entry. - pub fn new(name: impl Into, port: Option) -> Self { - Self { - domain: name.into(), - port, - } - } - - /// An allow-list entry that specifies a host and allows the default port. - pub fn host(name: impl Into) -> Self { - Self { - domain: name.into(), - port: None, - } - } - - /// An allow-list entry that specifies a host and port. - pub fn host_and_port(name: impl Into, port: u16) -> Self { - Self { - domain: name.into(), - port: Some(port), - } - } - - fn allow(&self, url: &url::Url) -> bool { - (url.scheme() == "http" || url.scheme() == "https") - && self.domain == url.host_str().unwrap_or_default() - && self.port == url.port() - } -} - -impl From for String { - fn from(allowed: AllowedHttpHost) -> Self { - match allowed.port { - Some(port) => format!("{}:{}", allowed.domain, port), - None => allowed.domain, - } - } -} - /// WebAssembly configuration. #[derive(Clone, Debug, Default)] pub struct WasmConfig { @@ -240,7 +167,7 @@ pub struct WasmConfig { /// List of directory mounts that need to be mapped inside the WebAssembly module. pub mounts: Vec, /// Optional list of HTTP hosts the component is allowed to connect. - pub allowed_http_hosts: AllowedHttpHosts, + pub allowed_http_hosts: Vec, } /// Directory mount for the assets of a component. diff --git a/crates/outbound-http/Cargo.toml b/crates/outbound-http/Cargo.toml index 67a02c7db..b60c00f2d 100644 --- a/crates/outbound-http/Cargo.toml +++ b/crates/outbound-http/Cargo.toml @@ -9,16 +9,15 @@ doctest = false [dependencies] anyhow = "1.0" -bytes = "1" -futures = "0.3" http = "0.2" -reqwest = { version = "0.11", default-features = true, features = [ "json", "blocking" ] } -spin-engine = { path = "../engine" } -spin-manifest = { path = "../manifest" } +reqwest = "0.11" +spin-app = { path = "../app" } +spin-core = { path = "../core" } tracing = { version = "0.1", features = [ "log" ] } url = "2.2.1" [dependencies.wit-bindgen-wasmtime] git = "https://github.com/bytecodealliance/wit-bindgen" rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" -features = ["async"] \ No newline at end of file +features = ["async"] + diff --git a/crates/loader/src/validation.rs b/crates/outbound-http/src/allowed_http_hosts.rs similarity index 81% rename from crates/loader/src/validation.rs rename to crates/outbound-http/src/allowed_http_hosts.rs index a0f89099c..4378ac1cd 100644 --- a/crates/loader/src/validation.rs +++ b/crates/outbound-http/src/allowed_http_hosts.rs @@ -1,8 +1,71 @@ -#![deny(missing_docs)] - use anyhow::{anyhow, Result}; use reqwest::Url; -use spin_manifest::{AllowedHttpHost, AllowedHttpHosts}; + +const ALLOW_ALL_HOSTS: &str = "insecure:allow-all"; + +/// An HTTP host allow-list. +#[derive(Clone, Debug, Eq, PartialEq)] +pub enum AllowedHttpHosts { + /// All HTTP hosts are allowed (the "insecure:allow-all" value was present in the list) + AllowAll, + /// Only the specified hosts are allowed. + AllowSpecific(Vec), +} + +impl Default for AllowedHttpHosts { + fn default() -> Self { + Self::AllowSpecific(vec![]) + } +} + +impl AllowedHttpHosts { + /// Tests whether the given URL is allowed according to the allow-list. + pub fn allow(&self, url: &url::Url) -> bool { + match self { + Self::AllowAll => true, + Self::AllowSpecific(hosts) => hosts.iter().any(|h| h.allow(url)), + } + } +} + +/// An HTTP host allow-list entry. +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct AllowedHttpHost { + domain: String, + port: Option, +} + +impl AllowedHttpHost { + /// Creates a new allow-list entry. + pub fn new(name: impl Into, port: Option) -> Self { + Self { + domain: name.into(), + port, + } + } + + /// An allow-list entry that specifies a host and allows the default port. + pub fn host(name: impl Into) -> Self { + Self { + domain: name.into(), + port: None, + } + } + + /// An allow-list entry that specifies a host and port. + pub fn host_and_port(name: impl Into, port: u16) -> Self { + Self { + domain: name.into(), + port: Some(port), + } + } + + fn allow(&self, url: &url::Url) -> bool { + (url.scheme() == "http" || url.scheme() == "https") + && self.domain == url.host_str().unwrap_or_default() + && self.port == url.port() + } +} // Checks a list of allowed HTTP hosts is valid pub fn validate_allowed_http_hosts(http_hosts: &Option>) -> Result<()> { @@ -14,10 +77,7 @@ pub fn parse_allowed_http_hosts(raw: &Option>) -> Result Ok(AllowedHttpHosts::AllowSpecific(vec![])), Some(list) => { - if list - .iter() - .any(|domain| domain == outbound_http::ALLOW_ALL_HOSTS) - { + if list.iter().any(|domain| domain == ALLOW_ALL_HOSTS) { Ok(AllowedHttpHosts::AllowAll) } else { let parse_results = list diff --git a/crates/outbound-http/src/host_component.rs b/crates/outbound-http/src/host_component.rs index 935f50198..ecb857fbf 100644 --- a/crates/outbound-http/src/host_component.rs +++ b/crates/outbound-http/src/host_component.rs @@ -1,30 +1,39 @@ use anyhow::Result; -use wit_bindgen_wasmtime::wasmtime::Linker; -use spin_engine::{ - host_component::{HostComponent, HostComponentsStateHandle}, - RuntimeContext, -}; -use spin_manifest::CoreComponent; +use spin_app::DynamicHostComponent; +use spin_core::{Data, HostComponent, Linker}; -use crate::OutboundHttp; +use crate::{allowed_http_hosts::parse_allowed_http_hosts, OutboundHttp}; pub struct OutboundHttpComponent; +pub const ALLOWED_HTTP_HOSTS_METADATA_KEY: &str = "allowed_http_hosts"; + impl HostComponent for OutboundHttpComponent { - type State = OutboundHttp; + type Data = OutboundHttp; fn add_to_linker( - linker: &mut Linker>, - data_handle: HostComponentsStateHandle, + linker: &mut Linker, + get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, ) -> Result<()> { - crate::add_to_linker(linker, move |ctx| data_handle.get_mut(ctx))?; - Ok(()) + super::wasi_outbound_http::add_to_linker(linker, get) + } + + fn build_data(&self) -> Self::Data { + Default::default() } +} - fn build_state(&self, component: &CoreComponent) -> Result { - Ok(OutboundHttp { - allowed_hosts: component.wasm.allowed_http_hosts.clone(), - }) +impl DynamicHostComponent for OutboundHttpComponent { + fn update_data( + &self, + data: &mut Self::Data, + component: &spin_app::AppComponent, + ) -> anyhow::Result<()> { + let hosts = component + .get_metadata(ALLOWED_HTTP_HOSTS_METADATA_KEY) + .transpose()?; + data.allowed_hosts = parse_allowed_http_hosts(&hosts)?; + Ok(()) } } diff --git a/crates/outbound-http/src/lib.rs b/crates/outbound-http/src/lib.rs index 6dfbb1afb..b6fe1067f 100644 --- a/crates/outbound-http/src/lib.rs +++ b/crates/outbound-http/src/lib.rs @@ -1,19 +1,17 @@ +pub mod allowed_http_hosts; mod host_component; -use futures::executor::block_on; +use std::str::FromStr; + use http::HeaderMap; use reqwest::{Client, Url}; -use spin_manifest::AllowedHttpHosts; -use std::str::FromStr; -use wasi_outbound_http::*; -use wit_bindgen_wasmtime::async_trait; +use spin_app::async_trait; +use allowed_http_hosts::AllowedHttpHosts; pub use host_component::OutboundHttpComponent; -pub use wasi_outbound_http::add_to_linker; wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/wasi-outbound-http.wit"], async: *}); - -pub const ALLOW_ALL_HOSTS: &str = "insecure:allow-all"; +use wasi_outbound_http::*; /// A very simple implementation for outbound HTTP requests. #[derive(Default, Clone)] @@ -55,40 +53,38 @@ impl wasi_outbound_http::WasiOutboundHttp for OutboundHttp { } let client = Client::builder().build().unwrap(); - let res = client + let resp = client .request(method, url) .headers(headers) .body(body) .send() - .await; - let resp = log_request_error(res)?; - Response::try_from(resp).map_err(|_| HttpError::RuntimeError) + .await + .map_err(log_reqwest_error)?; + Response::from_reqwest(resp).await } } -fn log_request_error(response: Result) -> Result { - if let Err(e) = &response { - let error_desc = if e.is_timeout() { - "timeout error" - } else if e.is_connect() { - "connection error" - } else if e.is_body() || e.is_decode() { - "message body error" - } else if e.is_request() { - "request error" - } else { - "error" - }; - tracing::warn!( - "Outbound HTTP {}: URL {}, error detail {:?}", - error_desc, - e.url() - .map(|u| u.to_string()) - .unwrap_or_else(|| "".to_owned()), - e - ); - } - response +fn log_reqwest_error(err: reqwest::Error) -> HttpError { + let error_desc = if err.is_timeout() { + "timeout error" + } else if err.is_connect() { + "connection error" + } else if err.is_body() || err.is_decode() { + "message body error" + } else if err.is_request() { + "request error" + } else { + "error" + }; + tracing::warn!( + "Outbound HTTP {}: URL {}, error detail {:?}", + error_desc, + err.url() + .map(|u| u.to_string()) + .unwrap_or_else(|| "".to_owned()), + err + ); + HttpError::RuntimeError } impl From for http::Method { @@ -105,30 +101,12 @@ impl From for http::Method { } } -impl TryFrom for Response { - type Error = HttpError; - - fn try_from(res: reqwest::Response) -> Result { +impl Response { + async fn from_reqwest(res: reqwest::Response) -> Result { let status = res.status().as_u16(); let headers = response_headers(res.headers())?; - let body = Some(block_on(res.bytes())?.to_vec()); - - Ok(Response { - status, - headers, - body, - }) - } -} - -impl TryFrom for Response { - type Error = HttpError; - - fn try_from(res: reqwest::blocking::Response) -> Result { - let status = res.status().as_u16(); - let headers = response_headers(res.headers())?; - let body = Some(res.bytes()?.to_vec()); + let body = Some(res.bytes().await?.to_vec()); Ok(Response { status, diff --git a/crates/trigger-new/src/lib.rs b/crates/trigger-new/src/lib.rs index a229cf786..9425f3d4b 100644 --- a/crates/trigger-new/src/lib.rs +++ b/crates/trigger-new/src/lib.rs @@ -96,10 +96,10 @@ impl TriggerExecutorBuilder { // FIXME(lann): migrate host components from prototype // builder.add_host_component(outbound_redis::OutboundRedis::default())?; // builder.add_host_component(outbound_pg::OutboundPg)?; - // self.loader.add_dynamic_host_component( - // &mut builder, - // outbound_http::OutboundHttpComponent, - // )?; + self.loader.add_dynamic_host_component( + &mut builder, + outbound_http::OutboundHttpComponent, + )?; self.loader.add_dynamic_host_component( &mut builder, spin_config::ConfigHostComponent::new(self.default_config_providers(&app_uri)), diff --git a/crates/trigger-new/src/locked.rs b/crates/trigger-new/src/locked.rs index cd81e0b95..9e1d798e7 100644 --- a/crates/trigger-new/src/locked.rs +++ b/crates/trigger-new/src/locked.rs @@ -3,7 +3,6 @@ use std::path::{Path, PathBuf}; use anyhow::{anyhow, bail, Context, Result}; -use outbound_http::ALLOW_ALL_HOSTS; use spin_app::{ locked::{ self, ContentPath, ContentRef, LockedApp, LockedComponent, LockedComponentSource, @@ -12,8 +11,8 @@ use spin_app::{ values::{ValuesMap, ValuesMapBuilder}, }; use spin_manifest::{ - AllowedHttpHosts, Application, ApplicationInformation, ApplicationTrigger, CoreComponent, - HttpConfig, HttpTriggerConfiguration, RedisConfig, TriggerConfig, + Application, ApplicationInformation, ApplicationTrigger, CoreComponent, HttpConfig, + HttpTriggerConfiguration, RedisConfig, TriggerConfig, }; const WASM_CONTENT_TYPE: &str = "application/wasm"; @@ -130,10 +129,7 @@ impl LockedAppBuilder { let metadata = ValuesMapBuilder::new() .string_option("description", component.description) - .string_array( - "allowed_http_hosts", - allowed_http_hosts_to_strings(component.wasm.allowed_http_hosts), - ) + .string_array("allowed_http_hosts", component.wasm.allowed_http_hosts) .build(); let source = { @@ -197,13 +193,6 @@ fn file_uri(path: &Path, is_dir: bool) -> Result { Ok(uri.to_string()) } -fn allowed_http_hosts_to_strings(allowed_hosts: AllowedHttpHosts) -> Vec { - match allowed_hosts { - AllowedHttpHosts::AllowAll => vec![ALLOW_ALL_HOSTS.into()], - AllowedHttpHosts::AllowSpecific(hosts) => hosts.into_iter().map(Into::into).collect(), - } -} - #[cfg(test)] mod tests { use tempfile::TempDir; diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 3a04b6e47..97292a94d 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -126,7 +126,6 @@ impl TriggerExecutorBuilder { pub fn add_default_host_components( builder: &mut Builder, ) -> Result<()> { - builder.add_host_component(outbound_http::OutboundHttpComponent)?; builder.add_host_component(outbound_redis::OutboundRedis { connections: Arc::new(RwLock::new(HashMap::new())), })?; From 81a3629c210fc667102ba2205f2d2400cc012bd3 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 17:27:00 -0400 Subject: [PATCH 06/28] Migrate outbound-pg to new core Signed-off-by: Lann Martin --- Cargo.lock | 3 +-- crates/outbound-pg/Cargo.toml | 5 ++--- crates/outbound-pg/src/lib.rs | 29 ++++++++++------------------- crates/trigger-new/src/lib.rs | 2 +- crates/trigger/src/lib.rs | 3 --- 5 files changed, 14 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1775f6279..587d42194 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2237,8 +2237,7 @@ name = "outbound-pg" version = "0.2.0" dependencies = [ "anyhow", - "spin-engine", - "spin-manifest", + "spin-core", "tokio", "tokio-postgres", "tracing", diff --git a/crates/outbound-pg/Cargo.toml b/crates/outbound-pg/Cargo.toml index 44662cfe1..fa8d19bc2 100644 --- a/crates/outbound-pg/Cargo.toml +++ b/crates/outbound-pg/Cargo.toml @@ -8,10 +8,9 @@ doctest = false [dependencies] anyhow = "1.0" -tokio-postgres = { version = "0.7.7" } -spin-engine = { path = "../engine" } -spin-manifest = { path = "../manifest" } +spin-core = { path = "../core" } tokio = { version = "1", features = [ "rt-multi-thread" ] } +tokio-postgres = { version = "0.7.7" } tracing = { version = "0.1", features = [ "log" ] } [dependencies.wit-bindgen-wasmtime] diff --git a/crates/outbound-pg/src/lib.rs b/crates/outbound-pg/src/lib.rs index fc19eab91..d5940777d 100644 --- a/crates/outbound-pg/src/lib.rs +++ b/crates/outbound-pg/src/lib.rs @@ -1,42 +1,33 @@ use anyhow::anyhow; -use outbound_pg::*; +use spin_core::HostComponent; use std::collections::HashMap; use tokio_postgres::{ types::{ToSql, Type}, Client, NoTls, Row, }; - -pub use outbound_pg::add_to_linker; -use spin_engine::{ - host_component::{HostComponent, HostComponentsStateHandle}, - RuntimeContext, -}; -use wit_bindgen_wasmtime::{async_trait, wasmtime::Linker}; +use wit_bindgen_wasmtime::async_trait; wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/outbound-pg.wit"], async: *}); +use outbound_pg::{Column, DbDataType, DbValue, ParameterValue, PgError, RowSet}; /// A simple implementation to support outbound pg connection +#[derive(Default)] pub struct OutboundPg { pub connections: HashMap, } impl HostComponent for OutboundPg { - type State = Self; + type Data = Self; fn add_to_linker( - linker: &mut Linker>, - state_handle: HostComponentsStateHandle, + linker: &mut spin_core::Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, ) -> anyhow::Result<()> { - add_to_linker(linker, move |ctx| state_handle.get_mut(ctx)) + outbound_pg::add_to_linker(linker, get) } - fn build_state( - &self, - _component: &spin_manifest::CoreComponent, - ) -> anyhow::Result { - let connections = std::collections::HashMap::new(); - - Ok(Self { connections }) + fn build_data(&self) -> Self::Data { + Default::default() } } diff --git a/crates/trigger-new/src/lib.rs b/crates/trigger-new/src/lib.rs index 9425f3d4b..be3d16d1d 100644 --- a/crates/trigger-new/src/lib.rs +++ b/crates/trigger-new/src/lib.rs @@ -95,7 +95,7 @@ impl TriggerExecutorBuilder { if !self.disable_default_host_components { // FIXME(lann): migrate host components from prototype // builder.add_host_component(outbound_redis::OutboundRedis::default())?; - // builder.add_host_component(outbound_pg::OutboundPg)?; + builder.add_host_component(outbound_pg::OutboundPg::default())?; self.loader.add_dynamic_host_component( &mut builder, outbound_http::OutboundHttpComponent, diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 97292a94d..b983ae4d4 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -129,8 +129,5 @@ pub fn add_default_host_components( builder.add_host_component(outbound_redis::OutboundRedis { connections: Arc::new(RwLock::new(HashMap::new())), })?; - builder.add_host_component(outbound_pg::OutboundPg { - connections: HashMap::new(), - })?; Ok(()) } From a9de5f26a9b35208f248f027321476f52481a47a Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 12 Sep 2022 17:29:04 -0400 Subject: [PATCH 07/28] Migrate outbound-redis to new core Signed-off-by: Lann Martin --- Cargo.lock | 14 +--- crates/outbound-redis/Cargo.toml | 9 +-- crates/outbound-redis/src/lib.rs | 126 +++++++++++-------------------- crates/trigger-new/src/lib.rs | 3 +- crates/trigger/src/lib.rs | 13 +--- 5 files changed, 55 insertions(+), 110 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 587d42194..59a0fa1d0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2249,23 +2249,13 @@ name = "outbound-redis" version = "0.2.0" dependencies = [ "anyhow", - "owning_ref", "redis", - "spin-engine", - "spin-manifest", + "spin-core", + "tokio", "tracing", "wit-bindgen-wasmtime", ] -[[package]] -name = "owning_ref" -version = "0.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff55baddef9e4ad00f88b6c743a2a8062d4c6ade126c2a528644b8e444d52ce" -dependencies = [ - "stable_deref_trait", -] - [[package]] name = "parking_lot" version = "0.11.2" diff --git a/crates/outbound-redis/Cargo.toml b/crates/outbound-redis/Cargo.toml index 97b89fcec..8a1089152 100644 --- a/crates/outbound-redis/Cargo.toml +++ b/crates/outbound-redis/Cargo.toml @@ -8,11 +8,10 @@ doctest = false [dependencies] anyhow = "1.0" -owning_ref = "0.4.1" -redis = { version = "0.21", features = [ "tokio-comp" ] } -spin-engine = { path = "../engine" } -spin-manifest = { path = "../manifest" } -tracing = { version = "0.1", features = [ "log" ] } +redis = { version = "0.21", features = ["tokio-comp"] } +spin-core = { path = "../core" } +tokio = { version = "1", features = ["sync"] } +tracing = { version = "0.1", features = ["log"] } [dependencies.wit-bindgen-wasmtime] git = "https://github.com/bytecodealliance/wit-bindgen" diff --git a/crates/outbound-redis/src/lib.rs b/crates/outbound-redis/src/lib.rs index 71f3cd651..8309faf88 100644 --- a/crates/outbound-redis/src/lib.rs +++ b/crates/outbound-redis/src/lib.rs @@ -1,118 +1,84 @@ -use outbound_redis::*; -use owning_ref::RwLockReadGuardRef; -use redis::Commands; -use std::{ - collections::HashMap, - sync::{Arc, Mutex, RwLock}, -}; +use std::{collections::HashMap, sync::Arc}; -pub use outbound_redis::add_to_linker; -use spin_engine::{ - host_component::{HostComponent, HostComponentsStateHandle}, - RuntimeContext, -}; -use wit_bindgen_wasmtime::{async_trait, wasmtime::Linker}; +use anyhow::Result; +use redis::{aio::Connection, AsyncCommands}; +use spin_core::{HostComponent, Linker}; +use tokio::sync::{Mutex, RwLock}; +use wit_bindgen_wasmtime::async_trait; wit_bindgen_wasmtime::export!({paths: ["../../wit/ephemeral/outbound-redis.wit"], async: *}); +use outbound_redis::Error; -/// A simple implementation to support outbound Redis commands. +#[derive(Clone, Default)] pub struct OutboundRedis { - pub connections: Arc>>>, + connections: Arc>>>>, } impl HostComponent for OutboundRedis { - type State = Self; + type Data = Self; fn add_to_linker( - linker: &mut Linker>, - state_handle: HostComponentsStateHandle, + linker: &mut Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, ) -> anyhow::Result<()> { - add_to_linker(linker, move |ctx| state_handle.get_mut(ctx)) + crate::outbound_redis::add_to_linker(linker, get) } - fn build_state(&self, component: &spin_manifest::CoreComponent) -> anyhow::Result { - let mut conn_map = HashMap::new(); - if let Some(address) = component.wasm.environment.get("REDIS_ADDRESS") { - let client = redis::Client::open(address.to_string())?; - let conn = client.get_connection()?; - conn_map.insert(address.to_owned(), Mutex::new(conn)); - } - Ok(Self { - connections: Arc::new(RwLock::new(conn_map)), - }) + fn build_data(&self) -> Self::Data { + self.clone() } } -// TODO: use spawn_blocking or async client methods (redis::aio) #[async_trait] impl outbound_redis::OutboundRedis for OutboundRedis { async fn publish(&mut self, address: &str, channel: &str, payload: &[u8]) -> Result<(), Error> { - let conn_map = self.get_reused_conn_map(address)?; - let mut conn = conn_map - .get(address) - .unwrap() - .lock() - .map_err(|_| Error::Error)?; - conn.publish(channel, payload).map_err(|_| Error::Error)?; + let conn = self.get_conn(address).await.map_err(log_error)?; + conn.lock() + .await + .publish(channel, payload) + .await + .map_err(log_error)?; Ok(()) } async fn get(&mut self, address: &str, key: &str) -> Result, Error> { - let conn_map = self.get_reused_conn_map(address)?; - let mut conn = conn_map - .get(address) - .unwrap() - .lock() - .map_err(|_| Error::Error)?; - let value = conn.get(key).map_err(|_| Error::Error)?; + let conn = self.get_conn(address).await.map_err(log_error)?; + let value = conn.lock().await.get(key).await.map_err(log_error)?; Ok(value) } async fn set(&mut self, address: &str, key: &str, value: &[u8]) -> Result<(), Error> { - let conn_map = self.get_reused_conn_map(address)?; - let mut conn = conn_map - .get(address) - .unwrap() - .lock() - .map_err(|_| Error::Error)?; - conn.set(key, value).map_err(|_| Error::Error)?; + let conn = self.get_conn(address).await.map_err(log_error)?; + conn.lock().await.set(key, value).await.map_err(log_error)?; Ok(()) } async fn incr(&mut self, address: &str, key: &str) -> Result { - let conn_map = self.get_reused_conn_map(address)?; - let mut conn = conn_map - .get(address) - .unwrap() - .lock() - .map_err(|_| Error::Error)?; - let value = conn.incr(key, 1).map_err(|_| Error::Error)?; + let conn = self.get_conn(address).await.map_err(log_error)?; + let value = conn.lock().await.incr(key, 1).await.map_err(log_error)?; Ok(value) } } impl OutboundRedis { - fn get_reused_conn_map<'ret, 'me: 'ret, 'c>( - &'me mut self, - address: &'c str, - ) -> Result>>, Error> { - let conn_map = self.connections.read().map_err(|_| Error::Error)?; - if conn_map.get(address).is_some() { - tracing::debug!("Reuse connection: {:?}", address); - return Ok(RwLockReadGuardRef::new(conn_map)); - } - // Get rid of our read lock - drop(conn_map); - - let mut conn_map = self.connections.write().map_err(|_| Error::Error)?; - let client = redis::Client::open(address).map_err(|_| Error::Error)?; - let conn = client.get_connection().map_err(|_| Error::Error)?; - tracing::debug!("Build new connection: {:?}", address); - conn_map.insert(address.to_string(), Mutex::new(conn)); - // Get rid of our write lock - drop(conn_map); - - let conn_map = self.connections.read().map_err(|_| Error::Error)?; - Ok(RwLockReadGuardRef::new(conn_map)) + async fn get_conn(&self, address: &str) -> Result>> { + let conn_map = self.connections.read().await; + let conn = if let Some(conn) = conn_map.get(address) { + conn.clone() + } else { + let conn = redis::Client::open(address)?.get_async_connection().await?; + let conn = Arc::new(Mutex::new(conn)); + self.connections + .write() + .await + .insert(address.to_string(), conn.clone()); + conn + }; + Ok(conn) } } + +fn log_error(err: impl std::fmt::Debug) -> Error { + tracing::warn!("Outbound Redis error: {err:?}"); + Error::Error +} diff --git a/crates/trigger-new/src/lib.rs b/crates/trigger-new/src/lib.rs index be3d16d1d..f8b6924e3 100644 --- a/crates/trigger-new/src/lib.rs +++ b/crates/trigger-new/src/lib.rs @@ -93,8 +93,7 @@ impl TriggerExecutorBuilder { let mut builder = Engine::builder(&self.config)?; if !self.disable_default_host_components { - // FIXME(lann): migrate host components from prototype - // builder.add_host_component(outbound_redis::OutboundRedis::default())?; + builder.add_host_component(outbound_redis::OutboundRedis::default())?; builder.add_host_component(outbound_pg::OutboundPg::default())?; self.loader.add_dynamic_host_component( &mut builder, diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index b983ae4d4..1f3ca3b2c 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -1,10 +1,4 @@ -use std::{ - collections::HashMap, - error::Error, - marker::PhantomData, - path::PathBuf, - sync::{Arc, RwLock}, -}; +use std::{error::Error, marker::PhantomData, path::PathBuf}; use anyhow::Result; use async_trait::async_trait; @@ -124,10 +118,7 @@ impl TriggerExecutorBuilder { /// Add the default set of host components to the given builder. pub fn add_default_host_components( - builder: &mut Builder, + _builder: &mut Builder, ) -> Result<()> { - builder.add_host_component(outbound_redis::OutboundRedis { - connections: Arc::new(RwLock::new(HashMap::new())), - })?; Ok(()) } From 548efa3119a7740b4e8507cf14db2b539b96eb79 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 11:03:46 -0400 Subject: [PATCH 08/28] Migrate spin-http to new core Signed-off-by: Lann Martin --- Cargo.lock | 39 ++--- Cargo.toml | 1 + crates/app/src/lib.rs | 18 ++- crates/core/src/lib.rs | 2 +- crates/http/Cargo.toml | 13 +- crates/http/benches/baseline.rs | 15 +- crates/http/src/lib.rs | 178 ++++++++++----------- crates/http/src/routes.rs | 16 +- crates/http/src/spin.rs | 56 +++---- crates/http/src/wagi.rs | 157 ++++++++---------- crates/outbound-http/src/host_component.rs | 4 +- crates/testing/Cargo.toml | 5 + crates/testing/src/lib.rs | 128 +++++++++++++-- crates/trigger-new/src/lib.rs | 4 +- src/bin/spin.rs | 2 +- 15 files changed, 341 insertions(+), 297 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 59a0fa1d0..386816e54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1945,28 +1945,6 @@ dependencies = [ "unicase", ] -[[package]] -name = "mini-internal" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a63337614a1d280fdb2880599af563c99e9f388757f8d6515d785d85d14fb01" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "miniserde" -version = "0.1.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4313e4a66a442473e181963daf8c1e9def85c2d9fb0bb2ae59444260b28285" -dependencies = [ - "itoa 1.0.3", - "mini-internal", - "ryu", -] - [[package]] name = "miniz_oxide" version = "0.5.3" @@ -3368,6 +3346,7 @@ dependencies = [ "spin-redis-engine", "spin-templates", "spin-trigger", + "spin-trigger-new", "tempfile", "tokio", "toml", @@ -3442,20 +3421,19 @@ dependencies = [ "http", "hyper", "indexmap", - "miniserde", "num_cpus", "percent-encoding", "rustls-pemfile 0.3.0", - "spin-engine", - "spin-manifest", + "serde", + "serde_json", + "spin-app", + "spin-core", "spin-testing", - "spin-trigger", + "spin-trigger-new", "tls-listener", "tokio", "tokio-rustls", "tracing", - "wasi-common", - "wasmtime", "wit-bindgen-wasmtime", ] @@ -3629,10 +3607,15 @@ dependencies = [ "anyhow", "http", "hyper", + "serde", + "serde_json", + "spin-app", + "spin-core", "spin-engine", "spin-http", "spin-manifest", "spin-trigger", + "spin-trigger-new", "tracing-subscriber", ] diff --git a/Cargo.toml b/Cargo.toml index 748b50cec..0eb856338 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -41,6 +41,7 @@ spin-publish = { path = "crates/publish" } spin-redis-engine = { path = "crates/redis" } spin-templates = { path = "crates/templates" } spin-trigger = { path = "crates/trigger" } +spin-trigger-new = { path = "crates/trigger-new" } tempfile = "3.3.0" tokio = { version = "1.11", features = [ "full" ] } toml = "0.5" diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 89b099bdd..f0bc26b4e 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -110,16 +110,17 @@ impl<'a> App<'a> { &self.uri } - pub fn get_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Option> { + pub fn get_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result> { self.locked .metadata .get(key) .map(|value| Ok(T::deserialize(value)?)) + .transpose() } pub fn require_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result { - self.get_metadata(key) - .ok_or_else(|| Error::ManifestError(format!("missing required {key:?}")))? + self.get_metadata(key)? + .ok_or_else(|| Error::ManifestError(format!("missing required {key:?}"))) } pub fn variables(&self) -> impl Iterator { @@ -169,11 +170,18 @@ impl<'a> AppComponent<'a> { self.locked.files.iter() } - pub fn get_metadata>(&self, key: &str) -> Option> { + pub fn get_metadata>(&self, key: &str) -> Result> { self.locked .metadata .get(key) - .map(|value| Ok(T::deserialize(value)?)) + .map(|value| { + T::deserialize(value).map_err(|err| { + Error::ManifestError(format!( + "failed to deserialize {key:?} = {value:?}: {err:?}" + )) + }) + }) + .transpose() } pub fn config(&self) -> impl Iterator { diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 274638b28..7bc6ce021 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -9,7 +9,7 @@ use anyhow::Result; use tracing::instrument; use wasmtime_wasi::WasiCtx; -pub use wasmtime::{self, Instance, Module}; +pub use wasmtime::{self, Instance, Module, Trap}; use self::host_component::{HostComponents, HostComponentsBuilder}; diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index 900ef6983..ef03132ca 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -17,9 +17,11 @@ http = "0.2" hyper = { version = "0.14", features = ["full"] } indexmap = "1" percent-encoding = "2" -spin-manifest = { path = "../manifest" } -spin-engine = { path = "../engine" } -spin-trigger = { path = "../trigger" } +rustls-pemfile = "0.3.0" +serde = { version = "1.0", features = ["derive"] } +spin-app = { path = "../app" } +spin-core = { path = "../core" } +spin-trigger-new = { path = "../trigger-new" } tls-listener = { version = "0.4.0", features = [ "rustls", "hyper-h1", @@ -27,10 +29,7 @@ tls-listener = { version = "0.4.0", features = [ ] } tokio = { version = "1.10", features = ["full"] } tokio-rustls = { version = "0.23.2" } -rustls-pemfile = "0.3.0" tracing = { version = "0.1", features = ["log"] } -wasi-common = "0.39.1" -wasmtime = { version = "0.39.1", features = ["async"] } [dependencies.wit-bindgen-wasmtime] git = "https://github.com/bytecodealliance/wit-bindgen" @@ -39,8 +38,8 @@ features = ["async"] [dev-dependencies] criterion = { version = "0.3.5", features = ["async_tokio"] } -miniserde = "0.1" num_cpus = "1" +serde_json = "1" spin-testing = { path = "../testing" } [[bench]] diff --git a/crates/http/benches/baseline.rs b/crates/http/benches/baseline.rs index ec97e77c7..00f625074 100644 --- a/crates/http/benches/baseline.rs +++ b/crates/http/benches/baseline.rs @@ -7,7 +7,6 @@ use futures::future::join_all; use http::uri::Scheme; use http::Request; use spin_http::HttpTrigger; -use spin_manifest::{HttpConfig, HttpExecutor}; use spin_testing::{assert_http_response_success, TestConfig}; use tokio::runtime::Runtime; use tokio::task; @@ -29,7 +28,7 @@ fn bench_startup(c: &mut Criterion) { b.to_async(&async_runtime).iter(|| async { let trigger = TestConfig::default() .test_program("spin-http-benchmark.wasm") - .http_trigger(Default::default()) + .http_spin_trigger("/") .build_http_trigger() .await; run_concurrent_requests(Arc::new(trigger), 0, 1).await; @@ -39,10 +38,7 @@ fn bench_startup(c: &mut Criterion) { b.to_async(&async_runtime).iter(|| async { let trigger = TestConfig::default() .test_program("wagi-benchmark.wasm") - .http_trigger(HttpConfig { - executor: Some(HttpExecutor::Wagi(Default::default())), - ..Default::default() - }) + .http_wagi_trigger("/", Default::default()) .build_http_trigger() .await; run_concurrent_requests(Arc::new(trigger), 0, 1).await; @@ -58,7 +54,7 @@ fn bench_spin_concurrency_minimal(c: &mut Criterion) { async_runtime.block_on( TestConfig::default() .test_program("spin-http-benchmark.wasm") - .http_trigger(Default::default()) + .http_spin_trigger("/") .build_http_trigger(), ), ); @@ -94,10 +90,7 @@ fn bench_wagi_concurrency_minimal(c: &mut Criterion) { async_runtime.block_on( TestConfig::default() .test_program("wagi-benchmark.wasm") - .http_trigger(HttpConfig { - executor: Some(HttpExecutor::Wagi(Default::default())), - ..Default::default() - }) + .http_wagi_trigger("/", Default::default()) .build_http_trigger(), ), ); diff --git a/crates/http/src/lib.rs b/crates/http/src/lib.rs index 6466a3fbd..929aec703 100644 --- a/crates/http/src/lib.rs +++ b/crates/http/src/lib.rs @@ -5,7 +5,7 @@ mod spin; mod tls; mod wagi; -use std::{future::ready, net::SocketAddr, path::PathBuf, sync::Arc}; +use std::{collections::HashMap, future::ready, net::SocketAddr, path::PathBuf, sync::Arc}; use anyhow::{Context, Error, Result}; use async_trait::async_trait; @@ -18,10 +18,8 @@ use hyper::{ service::{make_service_fn, service_fn}, Body, Request, Response, Server, }; -use spin_http::SpinHttpData; -use spin_manifest::{ComponentMap, HttpConfig, HttpTriggerConfiguration, TriggerConfig}; -use spin_trigger::TriggerExecutor; -pub use tls::TlsConfig; +use serde::{Deserialize, Serialize}; +use spin_trigger_new::{TriggerAppEngine, TriggerExecutor}; use tls_listener::TlsListener; use tokio::net::{TcpListener, TcpStream}; use tokio_rustls::server::TlsStream; @@ -32,27 +30,25 @@ use crate::{ spin::SpinHttpExecutor, wagi::WagiHttpExecutor, }; +pub use tls::TlsConfig; +pub use wagi::WagiTriggerConfig; wit_bindgen_wasmtime::import!({paths: ["../../wit/ephemeral/spin-http.wit"], async: *}); -type ExecutionContext = spin_engine::ExecutionContext; -type RuntimeContext = spin_engine::RuntimeContext; +pub(crate) type RuntimeData = spin_http::SpinHttpData; +pub(crate) type Store = spin_core::Store; + +/// App metadata key for storing HTTP trigger "base" value +pub const HTTP_BASE_METADATA_KEY: &str = "http_base"; /// The Spin HTTP trigger. -/// -/// Could this contain a list of multiple HTTP applications? -/// (there could be a field apps: HashMap, where -/// the key is the base path for the application, and the trigger -/// would work across multiple applications.) pub struct HttpTrigger { - /// Trigger configuration. - trigger_config: HttpTriggerConfiguration, - /// Component trigger configurations. - component_triggers: ComponentMap, - /// Router. + engine: TriggerAppEngine, router: Router, - /// Spin execution context. - engine: ExecutionContext, + // Base path for component routes. + base: String, + // Component ID -> component trigger config + component_trigger_configs: HashMap, } #[derive(Args)] @@ -83,45 +79,70 @@ impl CliArgs { } } -pub struct HttpTriggerConfig(String, HttpConfig); - -impl TryFrom<(String, TriggerConfig)> for HttpTriggerConfig { - type Error = spin_manifest::Error; +/// Configuration for the HTTP trigger +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct HttpTriggerConfig { + /// Component ID to invoke + pub component: String, + /// HTTP route the component will be invoked for + pub route: String, + /// The HTTP executor the component requires + #[serde(default)] + pub executor: Option, +} - fn try_from((component, config): (String, TriggerConfig)) -> Result { - Ok(HttpTriggerConfig(component, config.try_into()?)) - } +/// The executor for the HTTP component. +/// The component can either implement the Spin HTTP interface, +/// or the Wagi CGI interface. +/// +/// If an executor is not specified, the inferred default is `HttpExecutor::Spin`. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields, rename_all = "lowercase", tag = "type")] +pub enum HttpExecutorType { + /// The component implements the Spin HTTP interface. + #[default] + Spin, + /// The component implements the Wagi CGI interface. + Wagi(WagiTriggerConfig), } #[async_trait] impl TriggerExecutor for HttpTrigger { - type GlobalConfig = HttpTriggerConfiguration; + const TRIGGER_TYPE: &'static str = "http"; + type RuntimeData = RuntimeData; type TriggerConfig = HttpTriggerConfig; type RunConfig = CliArgs; - type RuntimeContext = SpinHttpData; - - fn new( - execution_context: ExecutionContext, - global_config: Self::GlobalConfig, - trigger_configs: impl IntoIterator, - ) -> Result { - let component_triggers: ComponentMap = trigger_configs - .into_iter() - .map(|config| (config.0, config.1)) - .collect(); - let router = Router::build(&global_config.base, &component_triggers)?; + fn new(engine: TriggerAppEngine) -> Result { + let base = engine + .app() + .get_metadata(HTTP_BASE_METADATA_KEY)? + .unwrap_or("/") + .to_string(); + + let component_routes = engine + .trigger_configs() + .map(|(_, config)| (config.component.as_str(), config.route.as_str())); + + let router = Router::build(&base, component_routes)?; + log::trace!( "Constructed router for application {}: {:?}", - execution_context.config.label, + engine.app_name, router.routes ); + let component_trigger_configs = engine + .trigger_configs() + .map(|(_, config)| (config.component.clone(), config.clone())) + .collect(); + Ok(Self { - trigger_config: global_config, - component_triggers, + engine, router, - engine: execution_context, + base, + component_trigger_configs, }) } @@ -135,10 +156,10 @@ impl TriggerExecutor for HttpTrigger { println!("Serving {}", base_url); log::info!("Serving {}", base_url); println!("Available Routes:"); - for (route, component) in &self.router.routes { - println!(" {}: {}{}", component, base_url, route); - if let Some(component) = self.engine.components.get(component) { - if let Some(description) = &component.core.description { + for (route, component_id) in &self.router.routes { + println!(" {}: {}{}", component_id, base_url, route); + if let Some(component) = self.engine.app().get_component(component_id) { + if let Some(description) = component.get_metadata::<&str>("description")? { println!(" {}", description); } } @@ -165,7 +186,7 @@ impl HttpTrigger { log::info!( "Processing request for application {} on URI {}", - &self.engine.config.label, + &self.engine.app_name, req.uri() ); @@ -173,35 +194,25 @@ impl HttpTrigger { "/healthz" => Ok(Response::new(Body::from("OK"))), route => match self.router.route(route) { Ok(component_id) => { - let trigger = self.component_triggers.get(component_id).unwrap(); + let trigger = self.component_trigger_configs.get(component_id).unwrap(); - let executor = match &trigger.executor { - Some(i) => i, - None => &spin_manifest::HttpExecutor::Spin, - }; - - let follow = self - .engine - .config - .follow_components - .should_follow(component_id); + let executor = trigger.executor.as_ref().unwrap_or(&HttpExecutorType::Spin); let res = match executor { - spin_manifest::HttpExecutor::Spin => { + HttpExecutorType::Spin => { let executor = SpinHttpExecutor; executor .execute( &self.engine, component_id, - &self.trigger_config.base, + &self.base, &trigger.route, req, addr, - follow, ) .await } - spin_manifest::HttpExecutor::Wagi(wagi_config) => { + HttpExecutorType::Wagi(wagi_config) => { let executor = WagiHttpExecutor { wagi_config: wagi_config.clone(), }; @@ -209,11 +220,10 @@ impl HttpTrigger { .execute( &self.engine, component_id, - &self.trigger_config.base, + &self.base, &trigger.route, req, addr, - follow, ) .await } @@ -393,13 +403,12 @@ pub(crate) trait HttpExecutor: Clone + Send + Sync + 'static { #[allow(clippy::too_many_arguments)] async fn execute( &self, - engine: &ExecutionContext, - component: &str, + engine: &TriggerAppEngine, + component_id: &str, base: &str, raw_route: &str, req: Request, client_addr: SocketAddr, - follow: bool, ) -> Result>; } @@ -408,9 +417,7 @@ mod tests { use std::collections::BTreeMap; use anyhow::Result; - use spin_manifest::{HttpConfig, HttpExecutor}; use spin_testing::test_socket_addr; - use spin_trigger::TriggerExecutorBuilder; use super::*; @@ -526,15 +533,11 @@ mod tests { #[tokio::test] async fn test_spin_http() -> Result<()> { - let mut cfg = spin_testing::TestConfig::default(); - cfg.test_program("rust-http-test.wasm") - .http_trigger(HttpConfig { - route: "/test".to_string(), - executor: Some(HttpExecutor::Spin), - }); - let app = cfg.build_application(); - - let trigger: HttpTrigger = TriggerExecutorBuilder::new(app).build().await?; + let trigger = spin_testing::TestConfig::default() + .test_program("rust-http-test.wasm") + .http_spin_trigger("/test") + .build_http_trigger() + .await; let body = Body::from("Fermyon".as_bytes().to_vec()); let req = http::Request::post("https://myservice.fermyon.dev/test?abc=def") @@ -555,14 +558,11 @@ mod tests { #[tokio::test] async fn test_wagi_http() -> Result<()> { - let mut cfg = spin_testing::TestConfig::default(); - cfg.test_program("wagi-test.wasm").http_trigger(HttpConfig { - route: "/test".to_string(), - executor: Some(HttpExecutor::Wagi(Default::default())), - }); - let app = cfg.build_application(); - - let trigger: HttpTrigger = TriggerExecutorBuilder::new(app).build().await?; + let trigger = spin_testing::TestConfig::default() + .test_program("wagi-test.wasm") + .http_wagi_trigger("/test", Default::default()) + .build_http_trigger() + .await; let body = Body::from("Fermyon".as_bytes().to_vec()); let req = http::Request::builder() @@ -579,13 +579,13 @@ mod tests { assert_eq!(res.status(), StatusCode::OK); let body_bytes = hyper::body::to_bytes(res.into_body()).await.unwrap(); - #[derive(miniserde::Deserialize)] + #[derive(Deserialize)] struct Env { args: Vec, vars: BTreeMap, } let env: Env = - miniserde::json::from_str(std::str::from_utf8(body_bytes.as_ref()).unwrap()).unwrap(); + serde_json::from_str(std::str::from_utf8(body_bytes.as_ref()).unwrap()).unwrap(); assert_eq!(env.args, ["/test", "abc=def"]); assert_eq!(env.vars["HTTP_X_CUSTOM_FOO"], "bar".to_string()); diff --git a/crates/http/src/routes.rs b/crates/http/src/routes.rs index 0b8744b35..16dfa5aab 100644 --- a/crates/http/src/routes.rs +++ b/crates/http/src/routes.rs @@ -5,7 +5,6 @@ use anyhow::{Context, Result}; use http::Uri; use indexmap::IndexMap; -use spin_manifest::{ComponentMap, HttpConfig}; use std::{borrow::Cow, fmt}; /// Router for the HTTP trigger. @@ -17,17 +16,14 @@ pub struct Router { impl Router { /// Builds a router based on application configuration. - pub(crate) fn build( + pub(crate) fn build<'a>( base: &str, - component_http_configs: &ComponentMap, + component_routes: impl IntoIterator, ) -> Result { - let routes = component_http_configs - .iter() - .map(|(component_id, http_config)| { - ( - RoutePattern::from(base, &http_config.route), - component_id.to_string(), - ) + let routes = component_routes + .into_iter() + .map(|(component_id, route)| { + (RoutePattern::from(base, route), component_id.to_string()) }) .collect(); diff --git a/crates/http/src/spin.rs b/crates/http/src/spin.rs index 123f13f96..ed6a8e537 100644 --- a/crates/http/src/spin.rs +++ b/crates/http/src/spin.rs @@ -1,14 +1,15 @@ -use crate::{ - spin_http::{Method, SpinHttp}, - ExecutionContext, HttpExecutor, RuntimeContext, -}; +use std::{net::SocketAddr, str, str::FromStr}; + use anyhow::Result; use async_trait::async_trait; use hyper::{Body, Request, Response}; -use spin_engine::io::ModuleIoRedirects; -use std::{net::SocketAddr, str, str::FromStr}; -use tracing::log; -use wasmtime::{Instance, Store}; +use spin_core::Instance; +use spin_trigger_new::TriggerAppEngine; + +use crate::{ + spin_http::{Method, SpinHttp}, + HttpExecutor, HttpTrigger, Store, +}; #[derive(Clone)] pub struct SpinHttpExecutor; @@ -17,40 +18,25 @@ pub struct SpinHttpExecutor; impl HttpExecutor for SpinHttpExecutor { async fn execute( &self, - engine: &ExecutionContext, - component: &str, + engine: &TriggerAppEngine, + component_id: &str, base: &str, raw_route: &str, req: Request, _client_addr: SocketAddr, - follow: bool, ) -> Result> { - log::trace!( + tracing::trace!( "Executing request using the Spin executor for component {}", - component + component_id ); - let mior = ModuleIoRedirects::new(follow); + let (instance, store) = engine.prepare_instance(component_id).await?; - let (store, instance) = engine - .prepare_component(component, None, Some(mior.pipes), None, None) - .await?; - - let resp_result = Self::execute_impl(store, instance, base, raw_route, req) + let resp = Self::execute_impl(store, instance, base, raw_route, req) .await - .map_err(contextualise_err); - - let log_result = - engine.save_output_to_logs(mior.read_handles.read(), component, true, true); - - // Defer checking for failures until here so that the logging runs - // even if the guest code fails. (And when checking, check the guest - // result first, so that guest failures are returned in preference to - // log failures.) - let resp = resp_result?; - log_result?; + .map_err(contextualise_err)?; - log::info!( + tracing::info!( "Request finished, sending response with status code {}", resp.status() ); @@ -60,7 +46,7 @@ impl HttpExecutor for SpinHttpExecutor { impl SpinHttpExecutor { pub async fn execute_impl( - mut store: Store, + mut store: Store, instance: Instance, base: &str, raw_route: &str, @@ -72,7 +58,7 @@ impl SpinHttpExecutor { headers = Self::headers(&mut req, raw_route, base)?; } - let engine = SpinHttp::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?; + let http_instance = SpinHttp::new(&mut store, &instance, |data| data.as_mut())?; let (parts, bytes) = req.into_parts(); let bytes = hyper::body::to_bytes(bytes).await?.to_vec(); @@ -102,10 +88,10 @@ impl SpinHttpExecutor { body, }; - let resp = engine.handle_http_request(&mut store, req).await?; + let resp = http_instance.handle_http_request(&mut store, req).await?; if resp.status < 100 || resp.status > 600 { - log::error!("malformed HTTP status code"); + tracing::error!("malformed HTTP status code"); return Ok(Response::builder() .status(http::StatusCode::INTERNAL_SERVER_ERROR) .body(Body::empty())?); diff --git a/crates/http/src/wagi.rs b/crates/http/src/wagi.rs index d919edb46..164c9f520 100644 --- a/crates/http/src/wagi.rs +++ b/crates/http/src/wagi.rs @@ -1,38 +1,70 @@ mod util; -use crate::{routes::RoutePattern, ExecutionContext, HttpExecutor}; +use std::{io::Cursor, net::SocketAddr}; + use anyhow::Result; use async_trait::async_trait; -use hyper::{body, Body, Request, Response}; -use spin_engine::io::{ - redirect_to_mem_buffer, Follow, OutputBuffers, RedirectPipes, WriteDestinations, -}; -use spin_manifest::WagiConfig; -use std::{ - net::SocketAddr, - sync::{Arc, RwLock, RwLockReadGuard}, +use hyper::{ + body::{self}, + Body, Request, Response, }; -use tracing::log; -use wasi_common::pipe::{ReadPipe, WritePipe}; +use serde::{Deserialize, Serialize}; +use spin_core::Trap; +use spin_trigger_new::TriggerAppEngine; + +use crate::{routes::RoutePattern, HttpExecutor, HttpTrigger}; + +/// Wagi specific configuration for the http executor. +#[derive(Clone, Debug, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WagiTriggerConfig { + /// The name of the entrypoint. + #[serde(default)] + pub entrypoint: String, + + /// A string representation of the argv array. + /// + /// This should be a space-separate list of strings. The value + /// ${SCRIPT_NAME} will be replaced with the Wagi SCRIPT_NAME, + /// and the value ${ARGS} will be replaced with the query parameter + /// name/value pairs presented as args. For example, + /// `param1=val1¶m2=val2` will become `param1=val1 param2=val2`, + /// which will then be presented to the program as two arguments + /// in argv. + #[serde(default)] + pub argv: String, +} + +impl Default for WagiTriggerConfig { + fn default() -> Self { + /// This is the default Wagi entrypoint. + const WAGI_DEFAULT_ENTRYPOINT: &str = "_start"; + const WAGI_DEFAULT_ARGV: &str = "${SCRIPT_NAME} ${ARGS}"; + + Self { + entrypoint: WAGI_DEFAULT_ENTRYPOINT.to_owned(), + argv: WAGI_DEFAULT_ARGV.to_owned(), + } + } +} #[derive(Clone)] pub struct WagiHttpExecutor { - pub wagi_config: WagiConfig, + pub wagi_config: WagiTriggerConfig, } #[async_trait] impl HttpExecutor for WagiHttpExecutor { async fn execute( &self, - engine: &ExecutionContext, + engine: &TriggerAppEngine, component: &str, base: &str, raw_route: &str, req: Request, client_addr: SocketAddr, - follow: bool, ) -> Result> { - log::trace!( + tracing::trace!( "Executing request using the Wagi executor for component {}", component ); @@ -54,7 +86,7 @@ impl HttpExecutor for WagiHttpExecutor { let body = body::to_bytes(body).await?.to_vec(); let len = body.len(); - let (redirects, outputs) = Self::streams_from_body(body, follow); + // TODO // The default host and TLS fields are currently hard-coded. let mut headers = util::build_headers( @@ -83,14 +115,16 @@ impl HttpExecutor for WagiHttpExecutor { headers.insert(keys[1].to_string(), val); } - let (mut store, instance) = engine - .prepare_component( - component, - None, - Some(redirects), - Some(headers), - Some(argv.split(' ').map(|s| s.to_owned()).collect()), - ) + let mut store_builder = engine.store_builder(component)?; + + // Set up Wagi environment + store_builder.args(argv.split(' '))?; + store_builder.env(headers)?; + store_builder.stdin_pipe(Cursor::new(body)); + let mut stdout_buffer = store_builder.stdout_buffered(); + + let (instance, mut store) = engine + .prepare_instance_with_store(component, store_builder) .await?; let start = instance @@ -103,81 +137,18 @@ impl HttpExecutor for WagiHttpExecutor { ) })?; tracing::trace!("Calling Wasm entry point"); - let guest_result = start.call_async(&mut store, &[], &mut []).await; + start + .call_async(&mut store, &[], &mut []) + .await + .or_else(ignore_successful_proc_exit_trap)?; tracing::info!("Module execution complete"); - let log_result = engine.save_output_to_logs(outputs.read(), component, false, true); - - // Defer checking for failures until here so that the logging runs - // even if the guest code fails. (And when checking, check the guest - // result first, so that guest failures are returned in preference to - // log failures.) - guest_result.or_else(ignore_successful_proc_exit_trap)?; - log_result?; - - let stdout = outputs.stdout.read().unwrap(); - util::compose_response(&stdout) - } -} - -impl WagiHttpExecutor { - fn streams_from_body( - body: Vec, - follow_on_stderr: bool, - ) -> (RedirectPipes, WagiRedirectReadHandles) { - let stdin = ReadPipe::from(body); - - let stdout_buf = vec![]; - let stdout_lock = Arc::new(RwLock::new(stdout_buf)); - let stdout_pipe = WritePipe::from_shared(stdout_lock.clone()); - - let (stderr_pipe, stderr_lock) = redirect_to_mem_buffer(Follow::stderr(follow_on_stderr)); - - let rd = RedirectPipes::new( - Box::new(stdin), - Box::new(stdout_pipe), - Box::new(stderr_pipe), - ); - - let h = WagiRedirectReadHandles { - stdout: stdout_lock, - stderr: stderr_lock, - }; - - (rd, h) - } -} - -struct WagiRedirectReadHandles { - stdout: Arc>>, - stderr: Arc>, -} - -impl WagiRedirectReadHandles { - fn read(&self) -> impl OutputBuffers + '_ { - WagiRedirectReadHandlesLock { - stdout: self.stdout.read().unwrap(), - stderr: self.stderr.read().unwrap(), - } - } -} - -struct WagiRedirectReadHandlesLock<'a> { - stdout: RwLockReadGuard<'a, Vec>, - stderr: RwLockReadGuard<'a, WriteDestinations>, -} - -impl<'a> OutputBuffers for WagiRedirectReadHandlesLock<'a> { - fn stdout(&self) -> &[u8] { - &self.stdout - } - fn stderr(&self) -> &[u8] { - self.stderr.buffer() + util::compose_response(&stdout_buffer.take()) } } fn ignore_successful_proc_exit_trap(guest_err: anyhow::Error) -> Result<()> { - match guest_err.root_cause().downcast_ref::() { + match guest_err.root_cause().downcast_ref::() { Some(trap) => match trap.i32_exit_status() { Some(0) => Ok(()), _ => Err(guest_err), diff --git a/crates/outbound-http/src/host_component.rs b/crates/outbound-http/src/host_component.rs index ecb857fbf..575037a6e 100644 --- a/crates/outbound-http/src/host_component.rs +++ b/crates/outbound-http/src/host_component.rs @@ -30,9 +30,7 @@ impl DynamicHostComponent for OutboundHttpComponent { data: &mut Self::Data, component: &spin_app::AppComponent, ) -> anyhow::Result<()> { - let hosts = component - .get_metadata(ALLOWED_HTTP_HOSTS_METADATA_KEY) - .transpose()?; + let hosts = component.get_metadata(ALLOWED_HTTP_HOSTS_METADATA_KEY)?; data.allowed_hosts = parse_allowed_http_hosts(&hosts)?; Ok(()) } diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 646f23217..6f0afd3df 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -8,8 +8,13 @@ authors = ["Fermyon Engineering "] anyhow = "1.0" http = "0.2" hyper = "0.14" +serde = "1" +serde_json = "1" +spin-app = { path = "../app" } +spin-core = { path = "../core" } spin-engine = { path = "../engine" } spin-manifest = { path = "../manifest" } spin-http = { path = "../http" } spin-trigger = { path = "../trigger" } +spin-trigger-new = { path = "../trigger-new" } tracing-subscriber = "0.3" \ No newline at end of file diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index 57aa59851..b7b8d258f 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -9,12 +9,18 @@ use std::{ use http::Response; use hyper::Body; -use spin_http::HttpTrigger; +use serde::de::DeserializeOwned; +use spin_app::{ + async_trait, + locked::{LockedApp, LockedComponentSource}, + AppComponent, Loader, +}; +use spin_core::{Module, StoreBuilder}; +use spin_http::{HttpExecutorType, HttpTrigger, HttpTriggerConfig, WagiTriggerConfig}; use spin_manifest::{ Application, ApplicationInformation, ApplicationOrigin, ApplicationTrigger, CoreComponent, - HttpConfig, ModuleSource, RedisConfig, RedisTriggerConfiguration, SpinVersion, TriggerConfig, + ModuleSource, RedisConfig, RedisTriggerConfiguration, SpinVersion, TriggerConfig, }; -use spin_trigger::TriggerExecutorBuilder; /// Initialize a test writer for `tracing`, making its output compatible with libtest pub fn init_tracing() { @@ -24,11 +30,19 @@ pub fn init_tracing() { }) } +// Convenience wrapper for deserializing from literal JSON +macro_rules! from_json { + ($($json:tt)+) => { + serde_json::from_value(serde_json::json!($($json)+)).expect("valid json") + }; +} + #[derive(Default)] pub struct TestConfig { module_path: Option, application_trigger: Option, trigger_config: Option, + http_trigger_config: HttpTriggerConfig, } impl TestConfig { @@ -46,12 +60,6 @@ impl TestConfig { ) } - pub fn http_trigger(&mut self, config: HttpConfig) -> &mut Self { - self.application_trigger = Some(ApplicationTrigger::Http(Default::default())); - self.trigger_config = Some(TriggerConfig::Http(config)); - self - } - pub fn redis_trigger(&mut self, config: RedisConfig) -> &mut Self { self.application_trigger = Some(ApplicationTrigger::Redis(RedisTriggerConfiguration { address: "redis://localhost:6379".to_owned(), @@ -60,6 +68,28 @@ impl TestConfig { self } + pub fn http_spin_trigger(&mut self, route: impl Into) -> &mut Self { + self.http_trigger_config = HttpTriggerConfig { + component: "test-component".to_string(), + route: route.into(), + executor: None, + }; + self + } + + pub fn http_wagi_trigger( + &mut self, + route: impl Into, + wagi_config: WagiTriggerConfig, + ) -> &mut Self { + self.http_trigger_config = HttpTriggerConfig { + component: "test-component".to_string(), + route: route.into(), + executor: Some(HttpExecutorType::Wagi(wagi_config)), + }; + self + } + pub fn build_application_information(&self) -> ApplicationInformation { ApplicationInformation { spin_version: SpinVersion::V1, @@ -108,12 +138,86 @@ impl TestConfig { } } - pub async fn build_http_trigger(&self) -> HttpTrigger { - TriggerExecutorBuilder::new(self.build_application()) - .build() + pub fn build_locked_app(&self) -> LockedApp { + let components = from_json!([{ + "id": "test-component", + "source": { + "content_type": "application/wasm", + "digest": "test-source", + }, + }]); + let triggers = from_json!([ + { + "id": "test-http-trigger", + "trigger_type": "http", + "trigger_config": self.http_trigger_config, + }, + ]); + let metadata = from_json!({"name": "test-app", "redis_address": "test-redis-host"}); + let variables = Default::default(); + LockedApp { + spin_lock_version: spin_app::locked::FixedVersion, + components, + triggers, + metadata, + variables, + } + } + + pub fn build_loader(&self) -> impl Loader { + init_tracing(); + TestLoader { + app: self.build_locked_app(), + module_path: self.module_path.clone().expect("module path to be set"), + } + } + + pub async fn build_trigger(&self) -> Executor + where + Executor::TriggerConfig: DeserializeOwned, + { + spin_trigger_new::TriggerExecutorBuilder::new(self.build_loader()) + .build(TEST_APP_URI.to_string()) .await .unwrap() } + + pub async fn build_http_trigger(&self) -> HttpTrigger { + self.build_trigger().await + } +} + +const TEST_APP_URI: &str = "spin-test:"; + +struct TestLoader { + app: LockedApp, + module_path: PathBuf, +} + +#[async_trait] +impl Loader for TestLoader { + async fn load_app(&self, uri: &str) -> anyhow::Result { + assert_eq!(uri, TEST_APP_URI); + Ok(self.app.clone()) + } + + async fn load_module( + &self, + engine: &spin_core::wasmtime::Engine, + source: &LockedComponentSource, + ) -> anyhow::Result { + assert_eq!(source.content.digest.as_deref(), Some("test-source"),); + Module::from_file(engine, &self.module_path) + } + + async fn mount_files( + &self, + _store_builder: &mut StoreBuilder, + component: &AppComponent, + ) -> anyhow::Result<()> { + assert_eq!(component.files().len(), 0, "files testing not implemented"); + Ok(()) + } } pub fn test_socket_addr() -> SocketAddr { diff --git a/crates/trigger-new/src/lib.rs b/crates/trigger-new/src/lib.rs index f8b6924e3..979fc66bb 100644 --- a/crates/trigger-new/src/lib.rs +++ b/crates/trigger-new/src/lib.rs @@ -10,7 +10,7 @@ use std::{ }; use anyhow::{Context, Result}; -use async_trait::async_trait; +pub use async_trait::async_trait; use serde::de::DeserializeOwned; use spin_app::{App, AppLoader, AppTrigger, Loader, OwnedApp}; @@ -30,7 +30,7 @@ pub trait TriggerExecutor: Sized { type RunConfig; /// Create a new trigger executor. - fn new(app_engine: TriggerAppEngine) -> Result; + fn new(engine: TriggerAppEngine) -> Result; /// Run the trigger executor. async fn run(self, config: Self::RunConfig) -> Result<()>; diff --git a/src/bin/spin.rs b/src/bin/spin.rs index 277ae8c4c..a365155ce 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -54,7 +54,7 @@ enum SpinApp { #[derive(Subcommand)] enum TriggerCommands { - Http(TriggerExecutorCommand), + Http(spin_trigger_new::cli::TriggerExecutorCommand), Redis(TriggerExecutorCommand), } From 24eebf4ee6680ae11676d2cf0cea3b97b351721e Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 11:29:00 -0400 Subject: [PATCH 09/28] Migrate spin-redis to new core Signed-off-by: Lann Martin --- Cargo.lock | 14 ++-- Cargo.toml | 1 - crates/redis/Cargo.toml | 9 ++- crates/redis/src/lib.rs | 141 +++++++++++++------------------------- crates/redis/src/spin.rs | 47 +++++-------- crates/redis/src/tests.rs | 18 ++--- crates/testing/Cargo.toml | 4 +- crates/testing/src/lib.rs | 71 +++---------------- src/bin/spin.rs | 4 +- 9 files changed, 94 insertions(+), 215 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 386816e54..cb01312d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,7 +3345,6 @@ dependencies = [ "spin-publish", "spin-redis-engine", "spin-templates", - "spin-trigger", "spin-trigger-new", "tempfile", "tokio", @@ -3542,13 +3541,12 @@ dependencies = [ "async-trait", "futures", "redis", - "spin-engine", - "spin-manifest", + "serde", + "spin-app", + "spin-core", "spin-testing", - "spin-trigger", - "tokio", + "spin-trigger-new", "tracing", - "wasmtime", "wit-bindgen-wasmtime", ] @@ -3611,11 +3609,9 @@ dependencies = [ "serde_json", "spin-app", "spin-core", - "spin-engine", "spin-http", - "spin-manifest", - "spin-trigger", "spin-trigger-new", + "tokio", "tracing-subscriber", ] diff --git a/Cargo.toml b/Cargo.toml index 0eb856338..33e1b4027 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,6 @@ spin-plugins = { path = "crates/plugins" } spin-publish = { path = "crates/publish" } spin-redis-engine = { path = "crates/redis" } spin-templates = { path = "crates/templates" } -spin-trigger = { path = "crates/trigger" } spin-trigger-new = { path = "crates/trigger-new" } tempfile = "3.3.0" tokio = { version = "1.11", features = [ "full" ] } diff --git a/crates/redis/Cargo.toml b/crates/redis/Cargo.toml index 88b959d71..c57483f02 100644 --- a/crates/redis/Cargo.toml +++ b/crates/redis/Cargo.toml @@ -11,12 +11,12 @@ doctest = false anyhow = "1.0" async-trait = "0.1" futures = "0.3" -spin-engine = { path = "../engine" } -spin-manifest = { path = "../manifest" } -spin-trigger = { path = "../trigger" } +serde = "1" +spin-app = { path = "../app" } +spin-core = { path = "../core" } +spin-trigger-new = { path = "../trigger-new" } redis = { version = "0.21", features = [ "tokio-comp" ] } tracing = { version = "0.1", features = [ "log" ] } -wasmtime = "0.39.1" [dependencies.wit-bindgen-wasmtime] git = "https://github.com/bytecodealliance/wit-bindgen" @@ -25,4 +25,3 @@ features = ["async"] [dev-dependencies] spin-testing = { path = "../testing" } -tokio = { version = "1", features = [ "full" ] } diff --git a/crates/redis/src/lib.rs b/crates/redis/src/lib.rs index c9985b461..7eeff23ad 100644 --- a/crates/redis/src/lib.rs +++ b/crates/redis/src/lib.rs @@ -2,100 +2,81 @@ mod spin; -use std::{collections::HashMap, sync::Arc}; +use std::collections::HashMap; -use anyhow::Result; +use anyhow::{Context, Result}; use async_trait::async_trait; use futures::StreamExt; use redis::{Client, ConnectionLike}; -use spin_manifest::{ComponentMap, RedisConfig, RedisTriggerConfiguration, TriggerConfig}; -use spin_redis::SpinRedisData; -use spin_trigger::{cli::NoArgs, TriggerExecutor}; +use serde::{de::IgnoredAny, Deserialize, Serialize}; +use spin_trigger_new::{cli::NoArgs, TriggerAppEngine, TriggerExecutor}; use crate::spin::SpinRedisExecutor; wit_bindgen_wasmtime::import!({paths: ["../../wit/ephemeral/spin-redis.wit"], async: *}); -type ExecutionContext = spin_engine::ExecutionContext; -type RuntimeContext = spin_engine::RuntimeContext; +pub(crate) type RuntimeData = spin_redis::SpinRedisData; +pub(crate) type Store = spin_core::Store; /// The Spin Redis trigger. -#[derive(Clone)] pub struct RedisTrigger { - /// Trigger configuration. - trigger_config: RedisTriggerConfiguration, - /// Component trigger configurations. - component_triggers: ComponentMap, - /// Spin execution context. - engine: Arc, - /// Map from channel name to tuple of component name & index. - subscriptions: HashMap, + engine: TriggerAppEngine, + // Redis address to connect to + address: String, + // Mapping of subscription channels to component IDs + channel_components: HashMap, } -pub struct RedisTriggerConfig(String, RedisConfig); - -impl TryFrom<(String, TriggerConfig)> for RedisTriggerConfig { - type Error = spin_manifest::Error; - - fn try_from((component, config): (String, TriggerConfig)) -> Result { - Ok(RedisTriggerConfig(component, config.try_into()?)) - } +/// Redis trigger configuration. +#[derive(Clone, Debug, Default, Deserialize, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RedisTriggerConfig { + /// Component ID to invoke + pub component: String, + /// Channel to subscribe to + pub channel: String, + /// Trigger executor (currently unused) + #[serde(default, skip_serializing)] + pub executor: IgnoredAny, } #[async_trait] impl TriggerExecutor for RedisTrigger { - type GlobalConfig = RedisTriggerConfiguration; + const TRIGGER_TYPE: &'static str = "redis"; + type RuntimeData = RuntimeData; type TriggerConfig = RedisTriggerConfig; type RunConfig = NoArgs; - type RuntimeContext = SpinRedisData; - - fn new( - execution_context: ExecutionContext, - global_config: Self::GlobalConfig, - trigger_configs: impl IntoIterator, - ) -> Result { - let component_triggers: ComponentMap = trigger_configs - .into_iter() - .map(|config| (config.0, config.1)) - .collect(); - let subscriptions = execution_context - .config - .components - .iter() - .enumerate() - .filter_map(|(idx, component)| { - component_triggers - .get(&component.id) - .map(|redis_config| (redis_config.channel.clone(), idx)) - }) + + fn new(engine: TriggerAppEngine) -> Result { + let address = engine + .app() + .require_metadata("redis_address") + .context("Failed to configure Redis trigger")?; + + let channel_components = engine + .trigger_configs() + .map(|(_, config)| (config.channel.clone(), config.component.clone())) .collect(); Ok(Self { - trigger_config: global_config, - component_triggers, - engine: Arc::new(execution_context), - subscriptions, + engine, + address, + channel_components, }) } /// Run the Redis trigger indefinitely. async fn run(self, _config: Self::RunConfig) -> Result<()> { - let address = self.trigger_config.address.as_str(); + let address = &self.address; tracing::info!("Connecting to Redis server at {}", address); let mut client = Client::open(address.to_string())?; let mut pubsub = client.get_async_connection().await?.into_pubsub(); // Subscribe to channels - for (subscription, idx) in self.subscriptions.iter() { - let name = &self.engine.config.components[*idx].id; - tracing::info!( - "Subscribed component #{} ({}) to channel: {}", - idx, - name, - subscription - ); - pubsub.subscribe(subscription).await?; + for (channel, component) in self.channel_components.iter() { + tracing::info!("Subscribing component {component:?} to channel {channel:?}"); + pubsub.subscribe(channel).await?; } let mut stream = pubsub.on_message(); @@ -120,35 +101,12 @@ impl RedisTrigger { let channel = msg.get_channel_name(); tracing::info!("Received message on channel {:?}", channel); - if let Some(idx) = self.subscriptions.get(channel).copied() { - let component = &self.engine.config.components[idx]; - let executor = self - .component_triggers - .get(&component.id) - .and_then(|t| t.executor.clone()) - .unwrap_or_default(); - - let follow = self - .engine - .config - .follow_components - .should_follow(&component.id); - - match executor { - spin_manifest::RedisExecutor::Spin => { - tracing::trace!("Executing Spin Redis component {}", component.id); - let executor = SpinRedisExecutor; - executor - .execute( - &self.engine, - &component.id, - channel, - msg.get_payload_bytes(), - follow, - ) - .await? - } - }; + if let Some(component_id) = self.channel_components.get(channel) { + tracing::trace!("Executing Redis component {component_id:?}"); + let executor = SpinRedisExecutor; + executor + .execute(&self.engine, component_id, channel, msg.get_payload_bytes()) + .await? } else { tracing::debug!("No subscription found for {:?}", channel); } @@ -163,11 +121,10 @@ impl RedisTrigger { pub(crate) trait RedisExecutor: Clone + Send + Sync + 'static { async fn execute( &self, - engine: &ExecutionContext, - component: &str, + engine: &TriggerAppEngine, + component_id: &str, channel: &str, payload: &[u8], - follow: bool, ) -> Result<()>; } diff --git a/crates/redis/src/spin.rs b/crates/redis/src/spin.rs index d53c22ffb..9bb361ac8 100644 --- a/crates/redis/src/spin.rs +++ b/crates/redis/src/spin.rs @@ -1,8 +1,9 @@ -use crate::{spin_redis::SpinRedis, ExecutionContext, RedisExecutor, RuntimeContext}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use async_trait::async_trait; -use spin_engine::io::ModuleIoRedirects; -use wasmtime::{Instance, Store}; +use spin_core::Instance; +use spin_trigger_new::TriggerAppEngine; + +use crate::{spin_redis::SpinRedis, RedisExecutor, RedisTrigger, Store}; #[derive(Clone)] pub struct SpinRedisExecutor; @@ -11,52 +12,40 @@ pub struct SpinRedisExecutor; impl RedisExecutor for SpinRedisExecutor { async fn execute( &self, - engine: &ExecutionContext, - component: &str, + engine: &TriggerAppEngine, + component_id: &str, channel: &str, payload: &[u8], - follow: bool, ) -> Result<()> { - tracing::trace!( - "Executing request using the Spin executor for component {}", - component - ); - - let mior = ModuleIoRedirects::new(follow); + tracing::trace!("Executing request using the Spin executor for component {component_id}"); - let (store, instance) = engine - .prepare_component(component, None, Some(mior.pipes), None, None) - .await?; + let (instance, store) = engine.prepare_instance(component_id).await?; - let result = match Self::execute_impl(store, instance, channel, payload.to_vec()).await { + match Self::execute_impl(store, instance, channel, payload.to_vec()).await { Ok(()) => { tracing::trace!("Request finished OK"); Ok(()) } Err(e) => { - tracing::trace!("Request finished with error {}", e); + tracing::trace!("Request finished with error {e}"); Err(e) } - }; - - let log_result = - engine.save_output_to_logs(mior.read_handles.read(), component, true, true); - - result.and(log_result) + } } } impl SpinRedisExecutor { pub async fn execute_impl( - mut store: Store, + mut store: Store, instance: Instance, _channel: &str, payload: Vec, ) -> Result<()> { - let engine = SpinRedis::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?; - - let _res = engine.handle_redis_message(&mut store, &payload).await; + let engine = SpinRedis::new(&mut store, &instance, |data| data.as_mut())?; - Ok(()) + engine + .handle_redis_message(&mut store, &payload) + .await? + .map_err(|err| anyhow!("{err:?}")) } } diff --git a/crates/redis/src/tests.rs b/crates/redis/src/tests.rs index 3de19348d..fdd75f0fd 100644 --- a/crates/redis/src/tests.rs +++ b/crates/redis/src/tests.rs @@ -1,9 +1,7 @@ use super::*; use anyhow::Result; use redis::{Msg, Value}; -use spin_manifest::{RedisConfig, RedisExecutor}; -use spin_testing::TestConfig; -use spin_trigger::TriggerExecutorBuilder; +use spin_testing::{tokio, TestConfig}; fn create_trigger_event(channel: &str, payload: &str) -> redis::Msg { Msg::from_value(&redis::Value::Bulk(vec![ @@ -17,15 +15,11 @@ fn create_trigger_event(channel: &str, payload: &str) -> redis::Msg { #[ignore] #[tokio::test] async fn test_pubsub() -> Result<()> { - let mut cfg = TestConfig::default(); - cfg.test_program("redis-rust.wasm") - .redis_trigger(RedisConfig { - channel: "messages".to_string(), - executor: Some(RedisExecutor::Spin), - }); - let app = cfg.build_application(); - - let trigger: RedisTrigger = TriggerExecutorBuilder::new(app).build().await?; + let trigger: RedisTrigger = TestConfig::default() + .test_program("redis-rust.wasm") + .redis_trigger("messages") + .build_trigger() + .await; let msg = create_trigger_event("messages", "hello"); trigger.handle(msg).await?; diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 6f0afd3df..0a0bd1027 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -12,9 +12,7 @@ serde = "1" serde_json = "1" spin-app = { path = "../app" } spin-core = { path = "../core" } -spin-engine = { path = "../engine" } -spin-manifest = { path = "../manifest" } spin-http = { path = "../http" } -spin-trigger = { path = "../trigger" } spin-trigger-new = { path = "../trigger-new" } +tokio = { version = "1", features = ["macros", "rt"] } tracing-subscriber = "0.3" \ No newline at end of file diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index b7b8d258f..e6abc2e7e 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -17,10 +17,9 @@ use spin_app::{ }; use spin_core::{Module, StoreBuilder}; use spin_http::{HttpExecutorType, HttpTrigger, HttpTriggerConfig, WagiTriggerConfig}; -use spin_manifest::{ - Application, ApplicationInformation, ApplicationOrigin, ApplicationTrigger, CoreComponent, - ModuleSource, RedisConfig, RedisTriggerConfiguration, SpinVersion, TriggerConfig, -}; +use spin_trigger_new::{TriggerExecutor, TriggerExecutorBuilder}; + +pub use tokio; /// Initialize a test writer for `tracing`, making its output compatible with libtest pub fn init_tracing() { @@ -40,9 +39,8 @@ macro_rules! from_json { #[derive(Default)] pub struct TestConfig { module_path: Option, - application_trigger: Option, - trigger_config: Option, http_trigger_config: HttpTriggerConfig, + redis_channel: String, } impl TestConfig { @@ -60,14 +58,6 @@ impl TestConfig { ) } - pub fn redis_trigger(&mut self, config: RedisConfig) -> &mut Self { - self.application_trigger = Some(ApplicationTrigger::Redis(RedisTriggerConfiguration { - address: "redis://localhost:6379".to_owned(), - })); - self.trigger_config = Some(TriggerConfig::Redis(config)); - self - } - pub fn http_spin_trigger(&mut self, route: impl Into) -> &mut Self { self.http_trigger_config = HttpTriggerConfig { component: "test-component".to_string(), @@ -90,52 +80,9 @@ impl TestConfig { self } - pub fn build_application_information(&self) -> ApplicationInformation { - ApplicationInformation { - spin_version: SpinVersion::V1, - name: "test-app".to_string(), - version: "1.0.0".to_string(), - description: None, - authors: vec![], - trigger: self - .application_trigger - .clone() - .expect("http_trigger or redis_trigger required"), - namespace: None, - origin: ApplicationOrigin::File( - PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("fake_spin.toml"), - ), - } - } - - pub fn build_component(&self) -> CoreComponent { - let module_path = self - .module_path - .clone() - .expect("module_path or test_program required"); - CoreComponent { - source: ModuleSource::FileReference(module_path), - id: "test-component".to_string(), - description: None, - wasm: Default::default(), - config: Default::default(), - } - } - - pub fn build_application(&self) -> Application { - Application { - info: self.build_application_information(), - components: vec![self.build_component()], - component_triggers: [( - "test-component".to_string(), - self.trigger_config - .clone() - .expect("http_trigger or redis_trigger required"), - )] - .into_iter() - .collect(), - variables: Default::default(), - } + pub fn redis_trigger(&mut self, channel: impl Into) -> &mut Self { + self.redis_channel = channel.into(); + self } pub fn build_locked_app(&self) -> LockedApp { @@ -172,11 +119,11 @@ impl TestConfig { } } - pub async fn build_trigger(&self) -> Executor + pub async fn build_trigger(&self) -> Executor where Executor::TriggerConfig: DeserializeOwned, { - spin_trigger_new::TriggerExecutorBuilder::new(self.build_loader()) + TriggerExecutorBuilder::new(self.build_loader()) .build(TEST_APP_URI.to_string()) .await .unwrap() diff --git a/src/bin/spin.rs b/src/bin/spin.rs index a365155ce..4e8868fd7 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -8,7 +8,7 @@ use spin_cli::commands::{ }; use spin_http::HttpTrigger; use spin_redis_engine::RedisTrigger; -use spin_trigger::cli::TriggerExecutorCommand; +use spin_trigger_new::cli::TriggerExecutorCommand; #[tokio::main] async fn main() -> Result<(), Error> { @@ -54,7 +54,7 @@ enum SpinApp { #[derive(Subcommand)] enum TriggerCommands { - Http(spin_trigger_new::cli::TriggerExecutorCommand), + Http(TriggerExecutorCommand), Redis(TriggerExecutorCommand), } From d3e48dd800d304312afce1b016388ff7f1b20f94 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 11:50:37 -0400 Subject: [PATCH 10/28] Migrate spin-timer example to new core Signed-off-by: Lann Martin --- Cargo.lock | 11 +----- examples/spin-timer/Cargo.toml | 11 +----- examples/spin-timer/src/main.rs | 65 +++++++++++++++------------------ 3 files changed, 31 insertions(+), 56 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cb01312d8..1d8a70b23 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3620,20 +3620,11 @@ name = "spin-timer" version = "0.1.0" dependencies = [ "anyhow", - "async-trait", "chrono", - "env_logger", - "futures", - "log", - "spin-engine", - "spin-manifest", - "spin-trigger", + "spin-core", "tokio", "tracing", "tracing-subscriber", - "wasi-common", - "wasmtime", - "wasmtime-wasi", "wit-bindgen-wasmtime", ] diff --git a/examples/spin-timer/Cargo.toml b/examples/spin-timer/Cargo.toml index 34bd8e2fd..a7d84a4a4 100644 --- a/examples/spin-timer/Cargo.toml +++ b/examples/spin-timer/Cargo.toml @@ -6,20 +6,11 @@ edition = "2021" [dependencies] anyhow = "1.0" -async-trait = "0.1" chrono = "0.4" -env_logger = "0.9" -futures = "0.3" -log = { version = "0.4", default-features = false } -spin-engine = { path = "../../crates/engine" } -spin-manifest = { path = "../../crates/manifest" } -spin-trigger = { path = "../../crates/trigger" } +spin-core = { path = "../../crates/core" } tokio = { version = "1.14", features = [ "full" ] } tracing = { version = "0.1", features = [ "log" ] } tracing-subscriber = { version = "0.3.7", features = [ "env-filter" ] } -wasi-common = "0.39.1" -wasmtime = "0.39.1" -wasmtime-wasi = "0.39.1" [dependencies.wit-bindgen-wasmtime] git = "https://github.com/bytecodealliance/wit-bindgen" diff --git a/examples/spin-timer/src/main.rs b/examples/spin-timer/src/main.rs index 087439bae..3523b515e 100644 --- a/examples/spin-timer/src/main.rs +++ b/examples/spin-timer/src/main.rs @@ -1,15 +1,14 @@ // The wit_bindgen_wasmtime::import below is triggering this lint. #![allow(clippy::needless_question_mark)] -use std::{sync::Arc, time::Duration}; +use std::time::Duration; use anyhow::Result; -use spin_engine::{Builder, ExecutionContextConfiguration}; -use spin_manifest::{CoreComponent, ModuleSource, WasmConfig}; +use spin_core::{Engine, InstancePre, Module}; wit_bindgen_wasmtime::import!({paths: ["spin-timer.wit"], async: *}); -type ExecutionContext = spin_engine::ExecutionContext; +type RuntimeData = spin_timer::SpinTimerData; #[tokio::main] async fn main() -> Result<()> { @@ -17,27 +16,32 @@ async fn main() -> Result<()> { .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) .init(); - let component = component(); - let engine = Builder::build_default(ExecutionContextConfiguration { - components: vec![component], - label: "timer-app".to_string(), - ..Default::default() - }) - .await?; + let engine = Engine::builder(&Default::default())?.build(); + + let module = Module::from_file( + engine.as_ref(), + "example/target/wasm32-wasi/release/rust_echo_test.wasm", + )?; + + let instance_pre = engine.instantiate_pre(&module)?; + let trigger = TimerTrigger { - engine: Arc::new(engine), + engine, + instance_pre, interval: Duration::from_secs(1), }; + trigger.run().await } -/// A custom timer trigger that executes the -/// first component of an application on every interval. -#[derive(Clone)] +/// A custom timer trigger that executes a component on +/// every interval. pub struct TimerTrigger { - /// The Spin execution context. - engine: Arc, - /// The interval at which the component is executed. + /// The Spin core engine. + pub engine: Engine, + /// The pre-initialized component instance to execute. + pub instance_pre: InstancePre, + /// The interval at which the component is executed. pub interval: Duration, } @@ -57,26 +61,15 @@ impl TimerTrigger { } /// Execute the first component in the application configuration. async fn handle(&self, msg: String) -> Result<()> { - let (mut store, instance) = self - .engine - .prepare_component(&self.engine.config.components[0].id, None, None, None, None) + let mut store = self.engine.store_builder().build()?; + let instance = self.instance_pre.instantiate_async(&mut store).await?; + let timer_instance = + spin_timer::SpinTimer::new(&mut store, &instance, |data| data.as_mut())?; + let res = timer_instance + .handle_timer_request(&mut store, &msg) .await?; - - let t = - spin_timer::SpinTimer::new(&mut store, &instance, |host| host.data.as_mut().unwrap())?; - let res = t.handle_timer_request(&mut store, &msg).await?; - log::info!("{}\n", res); + tracing::info!("{}\n", res); Ok(()) } } - -pub fn component() -> CoreComponent { - CoreComponent { - source: ModuleSource::FileReference("target/test-programs/echo.wasm".into()), - id: "test".to_string(), - description: None, - wasm: WasmConfig::default(), - config: Default::default(), - } -} From 6f4e0e6cafb145c60d08bcc1d9365872f39e1569 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 11:53:57 -0400 Subject: [PATCH 11/28] Rename spin-trigger-new crate to spin-trigger Signed-off-by: Lann Martin --- Cargo.lock | 28 +- Cargo.toml | 3 +- crates/http/Cargo.toml | 2 +- crates/http/src/lib.rs | 2 +- crates/http/src/spin.rs | 2 +- crates/http/src/wagi.rs | 2 +- crates/redis/Cargo.toml | 2 +- crates/redis/src/lib.rs | 2 +- crates/redis/src/spin.rs | 2 +- crates/testing/Cargo.toml | 2 +- crates/testing/src/lib.rs | 2 +- crates/trigger-new/Cargo.toml | 32 -- crates/trigger-new/src/cli.rs | 156 ---------- crates/trigger/Cargo.toml | 19 +- crates/trigger/src/cli.rs | 107 ++----- crates/trigger/src/lib.rs | 274 ++++++++++++++---- crates/{trigger-new => trigger}/src/loader.rs | 0 crates/{trigger-new => trigger}/src/locked.rs | 0 crates/{trigger-new => trigger}/src/stdio.rs | 0 src/bin/spin.rs | 2 +- 20 files changed, 275 insertions(+), 364 deletions(-) delete mode 100644 crates/trigger-new/Cargo.toml delete mode 100644 crates/trigger-new/src/cli.rs rename crates/{trigger-new => trigger}/src/loader.rs (100%) rename crates/{trigger-new => trigger}/src/locked.rs (100%) rename crates/{trigger-new => trigger}/src/stdio.rs (100%) diff --git a/Cargo.lock b/Cargo.lock index 1d8a70b23..89053d14a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3345,7 +3345,7 @@ dependencies = [ "spin-publish", "spin-redis-engine", "spin-templates", - "spin-trigger-new", + "spin-trigger", "tempfile", "tokio", "toml", @@ -3428,7 +3428,7 @@ dependencies = [ "spin-app", "spin-core", "spin-testing", - "spin-trigger-new", + "spin-trigger", "tls-listener", "tokio", "tokio-rustls", @@ -3545,7 +3545,7 @@ dependencies = [ "spin-app", "spin-core", "spin-testing", - "spin-trigger-new", + "spin-trigger", "tracing", "wit-bindgen-wasmtime", ] @@ -3610,7 +3610,7 @@ dependencies = [ "spin-app", "spin-core", "spin-http", - "spin-trigger-new", + "spin-trigger", "tokio", "tracing-subscriber", ] @@ -3631,26 +3631,6 @@ dependencies = [ [[package]] name = "spin-trigger" version = "0.2.0" -dependencies = [ - "anyhow", - "async-trait", - "clap 3.2.19", - "ctrlc", - "dotenvy", - "futures", - "outbound-http", - "outbound-pg", - "outbound-redis", - "spin-engine", - "spin-loader", - "spin-manifest", - "tracing", - "wasmtime", -] - -[[package]] -name = "spin-trigger-new" -version = "0.2.0" dependencies = [ "anyhow", "async-trait", diff --git a/Cargo.toml b/Cargo.toml index 33e1b4027..afd00c2c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -40,7 +40,7 @@ spin-plugins = { path = "crates/plugins" } spin-publish = { path = "crates/publish" } spin-redis-engine = { path = "crates/redis" } spin-templates = { path = "crates/templates" } -spin-trigger-new = { path = "crates/trigger-new" } +spin-trigger = { path = "crates/trigger" } tempfile = "3.3.0" tokio = { version = "1.11", features = [ "full" ] } toml = "0.5" @@ -86,7 +86,6 @@ members = [ "crates/templates", "crates/testing", "crates/trigger", - "crates/trigger-new", "examples/spin-timer", "sdk/rust", "sdk/rust/macro" diff --git a/crates/http/Cargo.toml b/crates/http/Cargo.toml index ef03132ca..517f1d3b8 100644 --- a/crates/http/Cargo.toml +++ b/crates/http/Cargo.toml @@ -21,7 +21,7 @@ rustls-pemfile = "0.3.0" serde = { version = "1.0", features = ["derive"] } spin-app = { path = "../app" } spin-core = { path = "../core" } -spin-trigger-new = { path = "../trigger-new" } +spin-trigger = { path = "../trigger" } tls-listener = { version = "0.4.0", features = [ "rustls", "hyper-h1", diff --git a/crates/http/src/lib.rs b/crates/http/src/lib.rs index 929aec703..dd1111ea6 100644 --- a/crates/http/src/lib.rs +++ b/crates/http/src/lib.rs @@ -19,7 +19,7 @@ use hyper::{ Body, Request, Response, Server, }; use serde::{Deserialize, Serialize}; -use spin_trigger_new::{TriggerAppEngine, TriggerExecutor}; +use spin_trigger::{TriggerAppEngine, TriggerExecutor}; use tls_listener::TlsListener; use tokio::net::{TcpListener, TcpStream}; use tokio_rustls::server::TlsStream; diff --git a/crates/http/src/spin.rs b/crates/http/src/spin.rs index ed6a8e537..603194fa9 100644 --- a/crates/http/src/spin.rs +++ b/crates/http/src/spin.rs @@ -4,7 +4,7 @@ use anyhow::Result; use async_trait::async_trait; use hyper::{Body, Request, Response}; use spin_core::Instance; -use spin_trigger_new::TriggerAppEngine; +use spin_trigger::TriggerAppEngine; use crate::{ spin_http::{Method, SpinHttp}, diff --git a/crates/http/src/wagi.rs b/crates/http/src/wagi.rs index 164c9f520..31c4446e3 100644 --- a/crates/http/src/wagi.rs +++ b/crates/http/src/wagi.rs @@ -10,7 +10,7 @@ use hyper::{ }; use serde::{Deserialize, Serialize}; use spin_core::Trap; -use spin_trigger_new::TriggerAppEngine; +use spin_trigger::TriggerAppEngine; use crate::{routes::RoutePattern, HttpExecutor, HttpTrigger}; diff --git a/crates/redis/Cargo.toml b/crates/redis/Cargo.toml index c57483f02..55cae827d 100644 --- a/crates/redis/Cargo.toml +++ b/crates/redis/Cargo.toml @@ -14,7 +14,7 @@ futures = "0.3" serde = "1" spin-app = { path = "../app" } spin-core = { path = "../core" } -spin-trigger-new = { path = "../trigger-new" } +spin-trigger = { path = "../trigger" } redis = { version = "0.21", features = [ "tokio-comp" ] } tracing = { version = "0.1", features = [ "log" ] } diff --git a/crates/redis/src/lib.rs b/crates/redis/src/lib.rs index 7eeff23ad..23f58a3bf 100644 --- a/crates/redis/src/lib.rs +++ b/crates/redis/src/lib.rs @@ -9,7 +9,7 @@ use async_trait::async_trait; use futures::StreamExt; use redis::{Client, ConnectionLike}; use serde::{de::IgnoredAny, Deserialize, Serialize}; -use spin_trigger_new::{cli::NoArgs, TriggerAppEngine, TriggerExecutor}; +use spin_trigger::{cli::NoArgs, TriggerAppEngine, TriggerExecutor}; use crate::spin::SpinRedisExecutor; diff --git a/crates/redis/src/spin.rs b/crates/redis/src/spin.rs index 9bb361ac8..e80686d00 100644 --- a/crates/redis/src/spin.rs +++ b/crates/redis/src/spin.rs @@ -1,7 +1,7 @@ use anyhow::{anyhow, Result}; use async_trait::async_trait; use spin_core::Instance; -use spin_trigger_new::TriggerAppEngine; +use spin_trigger::TriggerAppEngine; use crate::{spin_redis::SpinRedis, RedisExecutor, RedisTrigger, Store}; diff --git a/crates/testing/Cargo.toml b/crates/testing/Cargo.toml index 0a0bd1027..b46241525 100644 --- a/crates/testing/Cargo.toml +++ b/crates/testing/Cargo.toml @@ -13,6 +13,6 @@ serde_json = "1" spin-app = { path = "../app" } spin-core = { path = "../core" } spin-http = { path = "../http" } -spin-trigger-new = { path = "../trigger-new" } +spin-trigger = { path = "../trigger" } tokio = { version = "1", features = ["macros", "rt"] } tracing-subscriber = "0.3" \ No newline at end of file diff --git a/crates/testing/src/lib.rs b/crates/testing/src/lib.rs index e6abc2e7e..a040c3445 100644 --- a/crates/testing/src/lib.rs +++ b/crates/testing/src/lib.rs @@ -17,7 +17,7 @@ use spin_app::{ }; use spin_core::{Module, StoreBuilder}; use spin_http::{HttpExecutorType, HttpTrigger, HttpTriggerConfig, WagiTriggerConfig}; -use spin_trigger_new::{TriggerExecutor, TriggerExecutorBuilder}; +use spin_trigger::{TriggerExecutor, TriggerExecutorBuilder}; pub use tokio; diff --git a/crates/trigger-new/Cargo.toml b/crates/trigger-new/Cargo.toml deleted file mode 100644 index 3a304ec3f..000000000 --- a/crates/trigger-new/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -[package] -name = "spin-trigger-new" -version = "0.2.0" -edition = "2021" -authors = ["Fermyon Engineering "] - -[dependencies] -anyhow = "1.0" -async-trait = "0.1" -clap = { version = "3.1.15", features = ["derive", "env"] } -ctrlc = { version = "3.2", features = ["termination"] } -dirs = "4" -futures = "0.3" -outbound-http = { path = "../outbound-http" } -outbound-redis = { path = "../outbound-redis" } -outbound-pg = { path = "../outbound-pg" } -sanitize-filename = "0.4" -serde = "1.0" -serde_json = "1.0" -spin-app = { path = "../app" } -spin-config = { path = "../config" } -spin-core = { path = "../core" } -spin-loader = { path = "../loader" } -spin-manifest = { path = "../manifest" } -tracing = { version = "0.1", features = [ "log" ] } -url = "2" -wasmtime = "0.39.1" - -[dev-dependencies] -tempfile = "3.3.0" -toml = "0.5" -tokio = { version = "1.0", features = ["rt", "macros"] } \ No newline at end of file diff --git a/crates/trigger-new/src/cli.rs b/crates/trigger-new/src/cli.rs deleted file mode 100644 index 51db7aad1..000000000 --- a/crates/trigger-new/src/cli.rs +++ /dev/null @@ -1,156 +0,0 @@ -use std::path::PathBuf; - -use anyhow::{Context, Result}; -use clap::{Args, IntoApp, Parser}; -use serde::de::DeserializeOwned; - -use crate::{loader::TriggerLoader, stdio::FollowComponents}; -use crate::{TriggerExecutor, TriggerExecutorBuilder}; - -pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; -pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; -pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; -pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; - -// Set by `spin up` -pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; -pub const SPIN_WORKING_DIR: &str = "SPIN_WORKING_DIR"; - -/// A command that runs a TriggerExecutor. -#[derive(Parser, Debug)] -#[clap(next_help_heading = "TRIGGER OPTIONS")] -pub struct TriggerExecutorCommand -where - Executor::RunConfig: Args, -{ - /// Log directory for the stdout and stderr of components. - #[clap( - name = APP_LOG_DIR, - short = 'L', - long = "log-dir", - )] - pub log: Option, - - /// Disable Wasmtime cache. - #[clap( - name = DISABLE_WASMTIME_CACHE, - long = "disable-cache", - env = DISABLE_WASMTIME_CACHE, - conflicts_with = WASMTIME_CACHE_FILE, - takes_value = false, - )] - pub disable_cache: bool, - - /// Wasmtime cache configuration file. - #[clap( - name = WASMTIME_CACHE_FILE, - long = "cache", - env = WASMTIME_CACHE_FILE, - conflicts_with = DISABLE_WASMTIME_CACHE, - )] - pub cache: Option, - - /// Print output for given component(s) to stdout/stderr - #[clap( - name = FOLLOW_LOG_OPT, - long = "follow", - multiple_occurrences = true, - )] - pub follow_components: Vec, - - /// Print all component output to stdout/stderr - #[clap( - long = "follow-all", - conflicts_with = FOLLOW_LOG_OPT, - )] - pub follow_all_components: bool, - - /// Set the static assets of the components in the temporary directory as writable. - #[clap(long = "allow-transient-write")] - pub allow_transient_write: bool, - - #[clap(flatten)] - pub run_config: Executor::RunConfig, - - #[clap(long = "help-args-only", hide = true)] - pub help_args_only: bool, -} - -/// An empty implementation of clap::Args to be used as TriggerExecutor::RunConfig -/// for executors that do not need additional CLI args. -#[derive(Args)] -pub struct NoArgs; - -impl TriggerExecutorCommand -where - Executor::RunConfig: Args, - Executor::TriggerConfig: DeserializeOwned, -{ - /// Create a new TriggerExecutorBuilder from this TriggerExecutorCommand. - pub async fn run(self) -> Result<()> { - if self.help_args_only { - Self::command() - .disable_help_flag(true) - .help_template("{all-args}") - .print_long_help()?; - return Ok(()); - } - - // Required env vars - let working_dir = std::env::var(SPIN_WORKING_DIR).context(SPIN_WORKING_DIR)?; - let locked_url = std::env::var(SPIN_LOCKED_URL).context(SPIN_LOCKED_URL)?; - - let loader = TriggerLoader::new(working_dir, self.allow_transient_write); - - let executor: Executor = { - let mut builder = TriggerExecutorBuilder::new(loader); - self.update_wasmtime_config(builder.wasmtime_config_mut())?; - builder.follow_components(self.follow_components()); - if let Some(log_dir) = self.log { - builder.log_dir(log_dir); - } - builder.build(locked_url).await? - }; - - let run_fut = executor.run(self.run_config); - - let (abortable, abort_handle) = futures::future::abortable(run_fut); - ctrlc::set_handler(move || abort_handle.abort())?; - match abortable.await { - Ok(Ok(())) => { - tracing::info!("Trigger executor shut down: exiting"); - Ok(()) - } - Ok(Err(err)) => { - tracing::error!("Trigger executor failed: {:?}", err); - Err(err) - } - Err(_aborted) => { - tracing::info!("User requested shutdown: exiting"); - Ok(()) - } - } - } - - pub fn follow_components(&self) -> FollowComponents { - if self.follow_all_components { - FollowComponents::All - } else if self.follow_components.is_empty() { - FollowComponents::None - } else { - let followed = self.follow_components.clone().into_iter().collect(); - FollowComponents::Named(followed) - } - } - - fn update_wasmtime_config(&self, config: &mut spin_core::wasmtime::Config) -> Result<()> { - // Apply --cache / --disable-cache - if !self.disable_cache { - match &self.cache { - Some(p) => config.cache_config_load(p)?, - None => config.cache_config_load_default()?, - }; - } - Ok(()) - } -} diff --git a/crates/trigger/Cargo.toml b/crates/trigger/Cargo.toml index fdbadc7c6..b93f84b90 100644 --- a/crates/trigger/Cargo.toml +++ b/crates/trigger/Cargo.toml @@ -2,20 +2,31 @@ name = "spin-trigger" version = "0.2.0" edition = "2021" -authors = ["mossaka "] +authors = ["Fermyon Engineering "] [dependencies] anyhow = "1.0" async-trait = "0.1" clap = { version = "3.1.15", features = ["derive", "env"] } ctrlc = { version = "3.2", features = ["termination"] } -dotenvy = "0.15.1" +dirs = "4" futures = "0.3" outbound-http = { path = "../outbound-http" } outbound-redis = { path = "../outbound-redis" } outbound-pg = { path = "../outbound-pg" } -spin-engine = { path = "../engine" } +sanitize-filename = "0.4" +serde = "1.0" +serde_json = "1.0" +spin-app = { path = "../app" } +spin-config = { path = "../config" } +spin-core = { path = "../core" } spin-loader = { path = "../loader" } spin-manifest = { path = "../manifest" } tracing = { version = "0.1", features = [ "log" ] } -wasmtime = "0.39.1" \ No newline at end of file +url = "2" +wasmtime = "0.39.1" + +[dev-dependencies] +tempfile = "3.3.0" +toml = "0.5" +tokio = { version = "1.0", features = ["rt", "macros"] } \ No newline at end of file diff --git a/crates/trigger/src/cli.rs b/crates/trigger/src/cli.rs index e4ebf178e..51db7aad1 100644 --- a/crates/trigger/src/cli.rs +++ b/crates/trigger/src/cli.rs @@ -1,11 +1,10 @@ -use std::{error::Error, path::PathBuf}; +use std::path::PathBuf; -use anyhow::{bail, Context, Result}; +use anyhow::{Context, Result}; use clap::{Args, IntoApp, Parser}; -use spin_engine::io::FollowComponents; -use spin_loader::bindle::BindleConnectionInfo; -use spin_manifest::{Application, ApplicationTrigger, TriggerConfig}; +use serde::de::DeserializeOwned; +use crate::{loader::TriggerLoader, stdio::FollowComponents}; use crate::{TriggerExecutor, TriggerExecutorBuilder}; pub const APP_LOG_DIR: &str = "APP_LOG_DIR"; @@ -13,6 +12,10 @@ pub const DISABLE_WASMTIME_CACHE: &str = "DISABLE_WASMTIME_CACHE"; pub const FOLLOW_LOG_OPT: &str = "FOLLOW_ID"; pub const WASMTIME_CACHE_FILE: &str = "WASMTIME_CACHE_FILE"; +// Set by `spin up` +pub const SPIN_LOCKED_URL: &str = "SPIN_LOCKED_URL"; +pub const SPIN_WORKING_DIR: &str = "SPIN_WORKING_DIR"; + /// A command that runs a TriggerExecutor. #[derive(Parser, Debug)] #[clap(next_help_heading = "TRIGGER OPTIONS")] @@ -20,10 +23,6 @@ pub struct TriggerExecutorCommand where Executor::RunConfig: Args, { - /// Pass an environment variable (key=value) to all components of the application. - #[clap(long = "env", short = 'e', parse(try_from_str = parse_env_var))] - pub env: Vec<(String, String)>, - /// Log directory for the stdout and stderr of components. #[clap( name = APP_LOG_DIR, @@ -66,6 +65,10 @@ where )] pub follow_all_components: bool, + /// Set the static assets of the components in the temporary directory as writable. + #[clap(long = "allow-transient-write")] + pub allow_transient_write: bool, + #[clap(flatten)] pub run_config: Executor::RunConfig, @@ -81,11 +84,7 @@ pub struct NoArgs; impl TriggerExecutorCommand where Executor::RunConfig: Args, - Executor::GlobalConfig: TryFrom, - >::Error: Error + Send + Sync + 'static, - Executor::TriggerConfig: TryFrom<(String, TriggerConfig)>, - >::Error: - Error + Send + Sync + 'static, + Executor::TriggerConfig: DeserializeOwned, { /// Create a new TriggerExecutorBuilder from this TriggerExecutorCommand. pub async fn run(self) -> Result<()> { @@ -97,15 +96,22 @@ where return Ok(()); } - let app = self.build_application().await?; - let mut builder = TriggerExecutorBuilder::new(app); - self.update_wasmtime_config(builder.wasmtime_config_mut())?; - builder.follow_components(self.follow_components()); - if let Some(log_dir) = self.log { - builder.log_dir(log_dir); - } + // Required env vars + let working_dir = std::env::var(SPIN_WORKING_DIR).context(SPIN_WORKING_DIR)?; + let locked_url = std::env::var(SPIN_LOCKED_URL).context(SPIN_LOCKED_URL)?; + + let loader = TriggerLoader::new(working_dir, self.allow_transient_write); + + let executor: Executor = { + let mut builder = TriggerExecutorBuilder::new(loader); + self.update_wasmtime_config(builder.wasmtime_config_mut())?; + builder.follow_components(self.follow_components()); + if let Some(log_dir) = self.log { + builder.log_dir(log_dir); + } + builder.build(locked_url).await? + }; - let executor: Executor = builder.build().await?; let run_fut = executor.run(self.run_config); let (abortable, abort_handle) = futures::future::abortable(run_fut); @@ -125,52 +131,6 @@ where } } } -} - -impl TriggerExecutorCommand -where - Executor::RunConfig: Args, -{ - pub async fn build_application(&self) -> Result { - let working_dir = std::env::var("SPIN_WORKING_DIR").context("SPIN_WORKING_DIR")?; - let manifest_url = std::env::var("SPIN_MANIFEST_URL").context("SPIN_MANIFEST_URL")?; - let allow_transient_write: bool = std::env::var("SPIN_ALLOW_TRANSIENT_WRITE") - .unwrap_or_else(|_| "false".to_string()) - .trim() - .parse() - .context("SPIN_ALLOW_TRANSIENT_WRITE")?; - - // TODO(lann): Find a better home for this; spin_loader? - let mut app = if let Some(manifest_file) = manifest_url.strip_prefix("file:") { - let bindle_connection = std::env::var("BINDLE_URL") - .ok() - .map(|url| BindleConnectionInfo::new(url, false, None, None)); - spin_loader::from_file( - manifest_file, - working_dir, - &bindle_connection, - allow_transient_write, - ) - .await? - } else if let Some(bindle_url) = manifest_url.strip_prefix("bindle+") { - let (bindle_server, bindle_id) = bindle_url - .rsplit_once("?id=") - .context("invalid bindle URL")?; - spin_loader::from_bindle(bindle_id, bindle_server, working_dir, allow_transient_write) - .await? - } else { - bail!("invalid SPIN_MANIFEST_URL {}", manifest_url); - }; - - // Apply --env to all components in the given app - for c in app.components.iter_mut() { - for (k, v) in self.env.iter().cloned() { - c.wasm.environment.insert(k, v); - } - } - - Ok(app) - } pub fn follow_components(&self) -> FollowComponents { if self.follow_all_components { @@ -183,7 +143,7 @@ where } } - fn update_wasmtime_config(&self, config: &mut wasmtime::Config) -> Result<()> { + fn update_wasmtime_config(&self, config: &mut spin_core::wasmtime::Config) -> Result<()> { // Apply --cache / --disable-cache if !self.disable_cache { match &self.cache { @@ -194,12 +154,3 @@ where Ok(()) } } - -// Parse the environment variables passed in `key=value` pairs. -fn parse_env_var(s: &str) -> Result<(String, String)> { - let parts: Vec<_> = s.splitn(2, '=').collect(); - if parts.len() != 2 { - bail!("Environment variable must be of the form `key=value`"); - } - Ok((parts[0].to_owned(), parts[1].to_owned())) -} diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 1f3ca3b2c..979fc66bb 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -1,39 +1,49 @@ -use std::{error::Error, marker::PhantomData, path::PathBuf}; +pub mod cli; +mod loader; +pub mod locked; +mod stdio; -use anyhow::Result; -use async_trait::async_trait; -use spin_engine::{ - io::FollowComponents, Builder, Engine, ExecutionContext, ExecutionContextConfiguration, +use std::{ + collections::HashMap, + marker::PhantomData, + path::{Path, PathBuf}, }; -use spin_manifest::{Application, ApplicationTrigger, TriggerConfig}; -pub mod cli; +use anyhow::{Context, Result}; +pub use async_trait::async_trait; +use serde::de::DeserializeOwned; + +use spin_app::{App, AppLoader, AppTrigger, Loader, OwnedApp}; +use spin_config::{provider::env::EnvProvider, Provider}; +use spin_core::{Config, Engine, EngineBuilder, Instance, InstancePre, Store, StoreBuilder}; + +use stdio::{ComponentStdioWriter, FollowComponents}; + +const SPIN_HOME: &str = ".spin"; +const SPIN_CONFIG_ENV_PREFIX: &str = "SPIN_APP"; + #[async_trait] pub trait TriggerExecutor: Sized { - type GlobalConfig; + const TRIGGER_TYPE: &'static str; + type RuntimeData: Default + Send + Sync + 'static; type TriggerConfig; type RunConfig; - type RuntimeContext: Default + Send + 'static; /// Create a new trigger executor. - fn new( - execution_context: ExecutionContext, - global_config: Self::GlobalConfig, - trigger_configs: impl IntoIterator, - ) -> Result; + fn new(engine: TriggerAppEngine) -> Result; /// Run the trigger executor. async fn run(self, config: Self::RunConfig) -> Result<()>; /// Make changes to the ExecutionContext using the given Builder. - fn configure_execution_context(_builder: &mut Builder) -> Result<()> { + fn configure_engine(_builder: &mut EngineBuilder) -> Result<()> { Ok(()) } } pub struct TriggerExecutorBuilder { - application: Application, - wasmtime_config: wasmtime::Config, + loader: AppLoader, + config: Config, log_dir: Option, follow_components: FollowComponents, disable_default_host_components: bool, @@ -42,10 +52,10 @@ pub struct TriggerExecutorBuilder { impl TriggerExecutorBuilder { /// Create a new TriggerExecutorBuilder with the given Application. - pub fn new(application: Application) -> Self { + pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { Self { - application, - wasmtime_config: Default::default(), + loader: AppLoader::new(loader), + config: Default::default(), log_dir: None, follow_components: Default::default(), disable_default_host_components: false, @@ -56,8 +66,8 @@ impl TriggerExecutorBuilder { /// !!!Warning!!! Using a custom Wasmtime Config is entirely unsupported; /// many configurations are likely to cause errors or unexpected behavior. #[doc(hidden)] - pub fn wasmtime_config_mut(&mut self) -> &mut wasmtime::Config { - &mut self.wasmtime_config + pub fn wasmtime_config_mut(&mut self) -> &mut spin_core::wasmtime::Config { + self.config.wasmtime_config() } pub fn log_dir(&mut self, log_dir: PathBuf) -> &mut Self { @@ -75,50 +85,198 @@ impl TriggerExecutorBuilder { self } - pub async fn build(self) -> Result + pub async fn build(mut self, app_uri: String) -> Result where - Executor::GlobalConfig: TryFrom, - >::Error: - Error + Send + Sync + 'static, - Executor::TriggerConfig: TryFrom<(String, TriggerConfig)>, - >::Error: - Error + Send + Sync + 'static, + Executor::TriggerConfig: DeserializeOwned, { - let app = self.application; - - // Build ExecutionContext - let ctx_config = ExecutionContextConfiguration { - components: app.components, - label: app.info.name, - log_dir: self.log_dir, - follow_components: self.follow_components, + let engine = { + let mut builder = Engine::builder(&self.config)?; + + if !self.disable_default_host_components { + builder.add_host_component(outbound_redis::OutboundRedis::default())?; + builder.add_host_component(outbound_pg::OutboundPg::default())?; + self.loader.add_dynamic_host_component( + &mut builder, + outbound_http::OutboundHttpComponent, + )?; + self.loader.add_dynamic_host_component( + &mut builder, + spin_config::ConfigHostComponent::new(self.default_config_providers(&app_uri)), + )?; + } + + Executor::configure_engine(&mut builder)?; + builder.build() }; - let engine = Engine::new(self.wasmtime_config)?; - let mut ctx_builder = Builder::with_engine(ctx_config, engine)?; - ctx_builder.link_defaults()?; - if !self.disable_default_host_components { - add_default_host_components(&mut ctx_builder)?; - } - Executor::configure_execution_context(&mut ctx_builder)?; - let execution_context = ctx_builder.build().await?; + let app = self.loader.load_owned_app(app_uri).await?; + let app_name = app.require_metadata("name")?; - // Build trigger configurations - let global_config = app.info.trigger.try_into()?; - let trigger_configs = app - .component_triggers - .into_iter() - .map(|(id, config)| Ok((id, config).try_into()?)) - .collect::>>()?; + let log_dir = { + let sanitized_app = sanitize_filename::sanitize(&app_name); + let parent_dir = match dirs::home_dir() { + Some(home) => home.join(SPIN_HOME), + None => PathBuf::new(), // "./" + }; + parent_dir.join(sanitized_app).join("logs") + }; + std::fs::create_dir_all(&log_dir)?; // Run trigger executor - Executor::new(execution_context, global_config, trigger_configs) + Executor::new( + TriggerAppEngine::new(engine, app_name, app, log_dir, self.follow_components).await?, + ) + } + + pub fn default_config_providers(&self, app_uri: &str) -> Vec> { + // EnvProvider + let dotenv_path = app_uri + .strip_prefix("file://") + .and_then(|path| Path::new(path).parent()) + .unwrap_or_else(|| Path::new(".")) + .join(".env"); + vec![Box::new(EnvProvider::new( + SPIN_CONFIG_ENV_PREFIX, + Some(dotenv_path), + ))] } } -/// Add the default set of host components to the given builder. -pub fn add_default_host_components( - _builder: &mut Builder, -) -> Result<()> { - Ok(()) +/// Execution context for a TriggerExecutor executing a particular App. +pub struct TriggerAppEngine { + /// Engine to be used with this executor. + pub engine: Engine, + /// Name of the app for e.g. logging. + pub app_name: String, + // An owned wrapper of the App. + app: OwnedApp, + // Log directory + log_dir: PathBuf, + // Component stdio follow config + follow_components: FollowComponents, + // Trigger configs for this trigger type, with order matching `app.triggers_with_type(Executor::TRIGGER_TYPE)` + trigger_configs: Vec, + // Map of {Component ID -> InstancePre} for each component. + component_instance_pres: HashMap>, +} + +impl TriggerAppEngine { + /// Returns a new TriggerAppEngine. May return an error if trigger config validation or + /// component pre-instantiation fails. + pub async fn new( + engine: Engine, + app_name: String, + app: OwnedApp, + log_dir: PathBuf, + follow_components: FollowComponents, + ) -> Result + where + ::TriggerConfig: DeserializeOwned, + { + let trigger_configs = app + .triggers_with_type(Executor::TRIGGER_TYPE) + .map(|trigger| { + trigger.typed_config().with_context(|| { + format!("invalid trigger configuration for {:?}", trigger.id()) + }) + }) + .collect::>()?; + + let mut component_instance_pres = HashMap::default(); + for component in app.components() { + let module = component.load_module(&engine).await?; + let instance_pre = engine.instantiate_pre(&module)?; + component_instance_pres.insert(component.id().to_string(), instance_pre); + } + + Ok(Self { + engine, + app_name, + app, + log_dir, + follow_components, + trigger_configs, + component_instance_pres, + }) + } + + /// Returns a reference to the App. + pub fn app(&self) -> &App { + &self.app + } + + /// Returns AppTriggers and typed TriggerConfigs for this executor type. + pub fn trigger_configs(&self) -> impl Iterator { + self.app + .triggers_with_type(Executor::TRIGGER_TYPE) + .zip(&self.trigger_configs) + } + + /// Returns a new StoreBuilder for the given component ID. + pub fn store_builder(&self, component_id: &str) -> Result { + let mut builder = self.engine.store_builder(); + + // Set up stdio logging + builder.stdout_pipe(self.component_stdio_writer(component_id, "stdout")?); + builder.stderr_pipe(self.component_stdio_writer(component_id, "stderr")?); + + Ok(builder) + } + + fn component_stdio_writer( + &self, + component_id: &str, + log_suffix: &str, + ) -> Result { + let sanitized_component_id = sanitize_filename::sanitize(component_id); + // e.g. + let log_path = self + .log_dir + .join(format!("{sanitized_component_id}_{log_suffix}.txt")); + let follow = self.follow_components.should_follow(component_id); + ComponentStdioWriter::new(&log_path, follow) + .with_context(|| format!("Failed to open log file {log_path:?}")) + } + + /// Returns a new Store and Instance for the given component ID. + pub async fn prepare_instance( + &self, + component_id: &str, + ) -> Result<(Instance, Store)> { + let store_builder = self.store_builder(component_id)?; + self.prepare_instance_with_store(component_id, store_builder) + .await + } + + /// Returns a new Store and Instance for the given component ID and StoreBuilder. + pub async fn prepare_instance_with_store( + &self, + component_id: &str, + mut store_builder: StoreBuilder, + ) -> Result<(Instance, Store)> { + // Look up AppComponent + let component = self.app.get_component(component_id).with_context(|| { + format!( + "app {:?} has no component {:?}", + self.app_name, component_id + ) + })?; + + // Build Store + component.apply_store_config(&mut store_builder).await?; + let mut store = store_builder.build()?; + + // Instantiate + let instance = self.component_instance_pres[component_id] + .instantiate_async(&mut store) + .await + .with_context(|| { + format!( + "app {:?} component {:?} instantiation failed", + self.app_name, component_id + ) + })?; + + Ok((instance, store)) + } } diff --git a/crates/trigger-new/src/loader.rs b/crates/trigger/src/loader.rs similarity index 100% rename from crates/trigger-new/src/loader.rs rename to crates/trigger/src/loader.rs diff --git a/crates/trigger-new/src/locked.rs b/crates/trigger/src/locked.rs similarity index 100% rename from crates/trigger-new/src/locked.rs rename to crates/trigger/src/locked.rs diff --git a/crates/trigger-new/src/stdio.rs b/crates/trigger/src/stdio.rs similarity index 100% rename from crates/trigger-new/src/stdio.rs rename to crates/trigger/src/stdio.rs diff --git a/src/bin/spin.rs b/src/bin/spin.rs index 4e8868fd7..277ae8c4c 100644 --- a/src/bin/spin.rs +++ b/src/bin/spin.rs @@ -8,7 +8,7 @@ use spin_cli::commands::{ }; use spin_http::HttpTrigger; use spin_redis_engine::RedisTrigger; -use spin_trigger_new::cli::TriggerExecutorCommand; +use spin_trigger::cli::TriggerExecutorCommand; #[tokio::main] async fn main() -> Result<(), Error> { From a41c40cad4fbadb8f24d2c797bb3ba5500258263 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 13:25:49 -0400 Subject: [PATCH 12/28] Remove 'allow_transient_write' from spin-loader This functionality is now handled in spin-core by mounting files with write capabilites disabled. Various tidying in the vicinity of removals. Signed-off-by: Lann Martin --- crates/loader/src/assets.rs | 13 ------- crates/loader/src/bindle/assets.rs | 56 ++++++++---------------------- crates/loader/src/bindle/mod.rs | 30 ++++------------ crates/loader/src/local/assets.rs | 43 ++++++----------------- crates/loader/src/local/mod.rs | 31 +++-------------- crates/loader/src/local/tests.rs | 10 +++--- crates/trigger/src/locked.rs | 2 +- 7 files changed, 42 insertions(+), 143 deletions(-) diff --git a/crates/loader/src/assets.rs b/crates/loader/src/assets.rs index fc06d885e..c8f708e54 100644 --- a/crates/loader/src/assets.rs +++ b/crates/loader/src/assets.rs @@ -90,19 +90,6 @@ pub fn file_sha256_digest_string(path: impl AsRef) -> std::io::Result, - allow_transient_write: bool, -) -> Result<()> { - let mut perms = tokio::fs::metadata(&path).await?.permissions(); - perms.set_readonly(!allow_transient_write); - tokio::fs::set_permissions(path, perms.clone()) - .await - .with_context(|| anyhow!("Cannot set permission {:?}", perms.clone()))?; - Ok(()) -} - #[cfg(test)] mod test { use super::*; diff --git a/crates/loader/src/bindle/assets.rs b/crates/loader/src/bindle/assets.rs index 101664b2b..0c6fb25e9 100644 --- a/crates/loader/src/bindle/assets.rs +++ b/crates/loader/src/bindle/assets.rs @@ -12,7 +12,7 @@ use tracing::log; use crate::file_sha256_digest_string; use crate::{ - assets::{change_file_permission, create_dir, ensure_under}, + assets::{create_dir, ensure_under}, bindle::utils::BindleReader, }; @@ -22,15 +22,12 @@ pub(crate) async fn prepare_component( parcels: &[Label], base_dst: impl AsRef, component: &str, - allow_transient_write: bool, ) -> Result { let copier = Copier { reader: reader.clone(), id: bindle_id.clone(), }; - copier - .prepare(parcels, base_dst, component, allow_transient_write) - .await + copier.prepare(parcels, base_dst, component).await } pub(crate) struct Copier { @@ -44,7 +41,6 @@ impl Copier { parcels: &[Label], base_dst: impl AsRef, component: &str, - allow_transient_write: bool, ) -> Result { log::info!( "Mounting files from '{}' to '{}'", @@ -54,56 +50,34 @@ impl Copier { let host = create_dir(&base_dst, component).await?; let guest = "/".to_string(); - self.copy_all(parcels, &host, allow_transient_write).await?; + self.copy_all(parcels, &host).await?; Ok(DirectoryMount { host, guest }) } - async fn copy_all( - &self, - parcels: &[Label], - dir: impl AsRef, - allow_transient_write: bool, - ) -> Result<()> { - match stream::iter( - parcels - .iter() - .map(|p| self.copy(p, &dir, allow_transient_write)), - ) - .buffer_unordered(crate::MAX_PARALLEL_ASSET_PROCESSING) - .filter_map(|r| future::ready(r.err())) - .map(|e| log::error!("{:?}", e)) - .count() - .await + async fn copy_all(&self, parcels: &[Label], dir: impl AsRef) -> Result<()> { + match stream::iter(parcels.iter().map(|p| self.copy(p, &dir))) + .buffer_unordered(crate::MAX_PARALLEL_ASSET_PROCESSING) + .filter_map(|r| future::ready(r.err())) + .map(|e| log::error!("{:?}", e)) + .count() + .await { 0 => Ok(()), n => bail!("Error copying assets: {} file(s) not copied", n), } } - async fn copy( - &self, - p: &Label, - dir: impl AsRef, - allow_transient_write: bool, - ) -> Result<()> { + async fn copy(&self, p: &Label, dir: impl AsRef) -> Result<()> { let to = dir.as_ref().join(&p.name); ensure_under(&dir, &to)?; if to.exists() { match check_existing_file(to.clone(), p).await { - Ok(true) => { - change_file_permission(&to, allow_transient_write).await?; - return Ok(()); - } - Ok(false) => { - // file exists but digest doesn't match, set it to writable first for further writing - let perms = tokio::fs::metadata(&to).await?.permissions(); - if perms.readonly() { - change_file_permission(&to, true).await?; - } - } + // Copy already exists + Ok(true) => return Ok(()), + Ok(false) => (), Err(err) => tracing::error!("Error verifying existing parcel: {}", err), } } @@ -144,8 +118,6 @@ impl Copier { })?; } - change_file_permission(&to, allow_transient_write).await?; - Ok(()) } } diff --git a/crates/loader/src/bindle/mod.rs b/crates/loader/src/bindle/mod.rs index 8674ab6c7..46fea820a 100644 --- a/crates/loader/src/bindle/mod.rs +++ b/crates/loader/src/bindle/mod.rs @@ -15,13 +15,11 @@ use std::path::Path; use anyhow::{anyhow, Context, Result}; use bindle::Invoice; use futures::future; - use outbound_http::allowed_http_hosts::validate_allowed_http_hosts; use spin_manifest::{ Application, ApplicationInformation, ApplicationOrigin, CoreComponent, ModuleSource, SpinVersion, WasmConfig, }; -use tracing::log; use crate::bindle::{ config::{RawAppManifest, RawComponentManifest}, @@ -35,19 +33,14 @@ pub use utils::SPIN_MANIFEST_MEDIA_TYPE; /// prepared application configuration consumable by a Spin execution context. /// If a directory is provided, use it as the base directory to expand the assets, /// otherwise create a new temporary directory. -pub async fn from_bindle( - id: &str, - url: &str, - base_dst: impl AsRef, - allow_transient_write: bool, -) -> Result { +pub async fn from_bindle(id: &str, url: &str, base_dst: impl AsRef) -> Result { // TODO // Handle Bindle authentication. let connection_info = BindleConnectionInfo::new(url, false, None, None); let client = connection_info.client()?; let reader = BindleReader::remote(&client, &id.parse()?); - prepare(id, url, &reader, base_dst, allow_transient_write).await + prepare(id, url, &reader, base_dst).await } /// Converts a Bindle invoice into Spin configuration. @@ -56,7 +49,6 @@ async fn prepare( url: &str, reader: &BindleReader, base_dst: impl AsRef, - allow_transient_write: bool, ) -> Result { // First, get the invoice from the Bindle server. let invoice = reader @@ -67,12 +59,12 @@ async fn prepare( // Then, reconstruct the application manifest from the parcels. let raw: RawAppManifest = toml::from_slice(&reader.get_parcel(&find_manifest(&invoice)?).await?)?; - log::trace!("Recreated manifest from bindle: {:?}", raw); + tracing::trace!("Recreated manifest from bindle: {:?}", raw); validate_raw_app_manifest(&raw)?; let info = info(&raw, &invoice, url); - log::trace!("Application information from bindle: {:?}", info); + tracing::trace!("Application information from bindle: {:?}", info); let component_triggers = raw .components .iter() @@ -81,7 +73,7 @@ async fn prepare( let components = future::join_all( raw.components .into_iter() - .map(|c| async { core(c, &invoice, reader, &base_dst, allow_transient_write).await }) + .map(|c| async { core(c, &invoice, reader, &base_dst).await }) .collect::>(), ) .await @@ -115,7 +107,6 @@ async fn core( invoice: &Invoice, reader: &BindleReader, base_dst: impl AsRef, - allow_transient_write: bool, ) -> Result { let bytes = reader .get_parcel(&raw.source) @@ -129,15 +120,8 @@ async fn core( Some(group) => { let parcels = parcels_in_group(invoice, &group); vec![ - assets::prepare_component( - reader, - &invoice.bindle.id, - &parcels, - base_dst, - &id, - allow_transient_write, - ) - .await?, + assets::prepare_component(reader, &invoice.bindle.id, &parcels, base_dst, &id) + .await?, ] } None => vec![], diff --git a/crates/loader/src/local/assets.rs b/crates/loader/src/local/assets.rs index 8ece1d50a..3bc345487 100644 --- a/crates/loader/src/local/assets.rs +++ b/crates/loader/src/local/assets.rs @@ -1,8 +1,6 @@ #![deny(missing_docs)] -use crate::assets::{ - change_file_permission, create_dir, ensure_all_under, ensure_under, to_relative, -}; +use crate::assets::{create_dir, ensure_all_under, ensure_under, to_relative}; use anyhow::{anyhow, bail, ensure, Context, Result}; use futures::{future, stream, StreamExt}; use spin_manifest::DirectoryMount; @@ -10,7 +8,6 @@ use std::{ path::{Path, PathBuf}, vec, }; -use tracing::log; use walkdir::WalkDir; use super::config::{RawDirectoryPlacement, RawFileMount}; @@ -22,10 +19,9 @@ pub(crate) async fn prepare_component( src: impl AsRef, base_dst: impl AsRef, id: &str, - allow_transient_write: bool, exclude_files: &[String], ) -> Result> { - log::info!( + tracing::info!( "Mounting files from '{}' to '{}'", src.as_ref().display(), base_dst.as_ref().display() @@ -34,7 +30,7 @@ pub(crate) async fn prepare_component( let files = collect(raw_mounts, exclude_files, src)?; let host = create_dir(&base_dst, id).await?; let guest = "/".to_string(); - copy_all(&files, &host, allow_transient_write).await?; + copy_all(&files, &host).await?; Ok(vec![DirectoryMount { guest, host }]) } @@ -173,7 +169,7 @@ fn collect_placement( /// Generate a vector of file mounts given a file pattern. fn collect_pattern(pattern: &str, rel: impl AsRef) -> Result> { let abs = rel.as_ref().join(pattern); - log::trace!("Resolving asset file pattern '{:?}'", abs); + tracing::trace!("Resolving asset file pattern '{:?}'", abs); let matches = glob::glob(&abs.to_string_lossy())?; let specifiers = matches @@ -186,53 +182,36 @@ fn collect_pattern(pattern: &str, rel: impl AsRef) -> Result, - allow_transient_write: bool, -) -> Result<()> { - let copy_futures = files.iter().map(|f| copy(f, &dir, allow_transient_write)); +async fn copy_all(files: &[FileMount], dir: impl AsRef) -> Result<()> { + let copy_futures = files.iter().map(|f| copy(f, &dir)); let errors = stream::iter(copy_futures) .buffer_unordered(crate::MAX_PARALLEL_ASSET_PROCESSING) .filter_map(|r| future::ready(r.err())) - .map(|e| log::error!("{:?}", e)) + .map(|e| tracing::error!("{e:?}")) .count() .await; ensure!( errors == 0, - "Error copying assets: {} file(s) not copied", - errors + "Error copying assets: {errors} file(s) not copied", ); Ok(()) } /// Copy a single file to the mount directory, setting it as read-only. -async fn copy(file: &FileMount, dir: impl AsRef, allow_transient_write: bool) -> Result<()> { +async fn copy(file: &FileMount, dir: impl AsRef) -> Result<()> { let from = &file.src; let to = dir.as_ref().join(&file.relative_dst); ensure_under(&dir.as_ref(), &to.as_path())?; - log::trace!( - "Copying asset file '{}' -> '{}'", - from.display(), - to.display() - ); + tracing::trace!("Copying asset file '{from:?}' -> '{to:?}'"); - tokio::fs::create_dir_all(to.parent().expect("Cannot copy to file '/'")).await?; - - // if destination file is read-only, set it to writable first - let metadata = tokio::fs::metadata(&to).await; - if metadata.is_ok() && metadata.unwrap().permissions().readonly() { - change_file_permission(&to, true).await?; - } + tokio::fs::create_dir_all(to.parent().context("Cannot copy to file '/'")?).await?; let _ = tokio::fs::copy(&from, &to) .await .with_context(|| anyhow!("Error copying asset file '{}'", from.display()))?; - change_file_permission(&to, allow_transient_write).await?; - Ok(()) } diff --git a/crates/loader/src/local/mod.rs b/crates/loader/src/local/mod.rs index aedf356dd..ad240acf6 100644 --- a/crates/loader/src/local/mod.rs +++ b/crates/loader/src/local/mod.rs @@ -33,7 +33,6 @@ pub async fn from_file( app: impl AsRef, base_dst: impl AsRef, bindle_connection: &Option, - allow_transient_write: bool, ) -> Result { let app = app .as_ref() @@ -42,14 +41,7 @@ pub async fn from_file( let manifest = raw_manifest_from_file(&app).await?; validate_raw_app_manifest(&manifest)?; - prepare_any_version( - manifest, - app, - base_dst, - bindle_connection, - allow_transient_write, - ) - .await + prepare_any_version(manifest, app, base_dst, bindle_connection).await } /// Reads the spin.toml file as a raw manifest. @@ -75,12 +67,9 @@ async fn prepare_any_version( src: impl AsRef, base_dst: impl AsRef, bindle_connection: &Option, - allow_transient_write: bool, ) -> Result { match raw { - RawAppManifestAnyVersion::V1(raw) => { - prepare(raw, src, base_dst, bindle_connection, allow_transient_write).await - } + RawAppManifestAnyVersion::V1(raw) => prepare(raw, src, base_dst, bindle_connection).await, } } @@ -116,7 +105,6 @@ async fn prepare( src: impl AsRef, base_dst: impl AsRef, bindle_connection: &Option, - allow_transient_write: bool, ) -> Result { let info = info(raw.info, &src); @@ -131,9 +119,7 @@ async fn prepare( let components = future::join_all( raw.components .into_iter() - .map(|c| async { - core(c, &src, &base_dst, bindle_connection, allow_transient_write).await - }) + .map(|c| async { core(c, &src, &base_dst, bindle_connection).await }) .collect::>(), ) .await @@ -161,7 +147,6 @@ async fn core( src: impl AsRef, base_dst: impl AsRef, bindle_connection: &Option, - allow_transient_write: bool, ) -> Result { let id = raw.id; @@ -209,15 +194,7 @@ async fn core( let mounts = match raw.wasm.files { Some(f) => { let exclude_files = raw.wasm.exclude_files.unwrap_or_default(); - assets::prepare_component( - &f, - src, - &base_dst, - &id, - allow_transient_write, - &exclude_files, - ) - .await? + assets::prepare_component(&f, src, &base_dst, &id, &exclude_files).await? } None => vec![], }; diff --git a/crates/loader/src/local/tests.rs b/crates/loader/src/local/tests.rs index 9ff8567bd..f355fc795 100644 --- a/crates/loader/src/local/tests.rs +++ b/crates/loader/src/local/tests.rs @@ -11,7 +11,7 @@ async fn test_from_local_source() -> Result<()> { let temp_dir = tempfile::tempdir()?; let dir = temp_dir.path(); - let app = from_file(MANIFEST, dir, &None, false).await?; + let app = from_file(MANIFEST, dir, &None).await?; assert_eq!(app.info.name, "spin-local-source-test"); assert_eq!(app.info.version, "1.0.0"); @@ -105,7 +105,7 @@ async fn test_invalid_manifest() -> Result<()> { let temp_dir = tempfile::tempdir()?; let dir = temp_dir.path(); - let app = from_file(MANIFEST, dir, &None, false).await; + let app = from_file(MANIFEST, dir, &None).await; let e = app.unwrap_err().to_string(); assert!( @@ -162,7 +162,7 @@ async fn test_duplicate_component_id_is_rejected() -> Result<()> { let temp_dir = tempfile::tempdir()?; let dir = temp_dir.path(); - let app = from_file(MANIFEST, dir, &None, false).await; + let app = from_file(MANIFEST, dir, &None).await; assert!( app.is_err(), @@ -184,7 +184,7 @@ async fn test_insecure_allow_all_with_invalid_url() -> Result<()> { let temp_dir = tempfile::tempdir()?; let dir = temp_dir.path(); - let app = from_file(MANIFEST, dir, &None, false).await; + let app = from_file(MANIFEST, dir, &None).await; assert!( app.is_ok(), @@ -200,7 +200,7 @@ async fn test_invalid_url_in_allowed_http_hosts_is_rejected() -> Result<()> { let temp_dir = tempfile::tempdir()?; let dir = temp_dir.path(); - let app = from_file(MANIFEST, dir, &None, false).await; + let app = from_file(MANIFEST, dir, &None).await; assert!(app.is_err(), "Expected allowed_http_hosts parsing error"); diff --git a/crates/trigger/src/locked.rs b/crates/trigger/src/locked.rs index 9e1d798e7..bd7e622bc 100644 --- a/crates/trigger/src/locked.rs +++ b/crates/trigger/src/locked.rs @@ -230,7 +230,7 @@ mod tests { let tempdir = TempDir::new().expect("tempdir"); std::env::set_current_dir(tempdir.path()).unwrap(); std::fs::write("spin.toml", TEST_MANIFEST).expect("write manifest"); - let app = spin_loader::local::from_file("spin.toml", &tempdir, &None, false) + let app = spin_loader::local::from_file("spin.toml", &tempdir, &None) .await .expect("load app"); (app, tempdir) From eea88f9f37f454062e5a49b613553576f4d2c49e Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 13:37:04 -0400 Subject: [PATCH 13/28] Update `spin up` to use new spin-trigger semantics Signed-off-by: Lann Martin --- src/commands/up.rs | 76 ++++++++++++++++++++++------------------------ 1 file changed, 37 insertions(+), 39 deletions(-) diff --git a/src/commands/up.rs b/src/commands/up.rs index 078f95af3..4e4a6aec6 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -8,6 +8,7 @@ use anyhow::{bail, Context, Result}; use clap::{CommandFactory, Parser}; use spin_loader::bindle::BindleConnectionInfo; use spin_manifest::ApplicationTrigger; +use spin_trigger::cli::{SPIN_LOCKED_URL, SPIN_WORKING_DIR}; use tempfile::TempDir; use crate::opts::*; @@ -77,14 +78,14 @@ pub struct UpCommand { )] pub insecure: bool, + /// Pass an environment variable (key=value) to all components of the application. + #[clap(short = 'e', long = "env", parse(try_from_str = parse_env_var))] + pub env: Vec<(String, String)>, + /// Temporary directory for the static assets of the components. #[clap(long = "temp")] pub tmp: Option, - /// Set the static assets of the components in the temporary directory as writable. - #[clap(long = "allow-transient-write")] - pub allow_transient_write: bool, - /// All other args, to be passed through to the trigger #[clap(hide = true)] pub trigger_args: Vec, @@ -92,6 +93,8 @@ pub struct UpCommand { impl UpCommand { pub async fn run(self) -> Result<()> { + // For displaying help, first print `spin up`'s own usage text, then + // attempt to load an app and print trigger-type-specific usage. let help = self.help; if help { Self::command() @@ -117,49 +120,43 @@ impl UpCommand { }; let working_dir = working_dir_holder.path(); - let app = match (&self.app, &self.bindle) { + let mut app = match (&self.app, &self.bindle) { (app, None) => { let manifest_file = app .as_deref() .unwrap_or_else(|| DEFAULT_MANIFEST_FILE.as_ref()); let bindle_connection = self.bindle_connection(); - spin_loader::from_file( - manifest_file, - working_dir, - &bindle_connection, - self.allow_transient_write, - ) - .await? + spin_loader::from_file(manifest_file, working_dir, &bindle_connection).await? } (None, Some(bindle)) => match &self.server { - Some(server) => { - spin_loader::from_bindle( - bindle, - server, - working_dir, - self.allow_transient_write, - ) - .await? - } + Some(server) => spin_loader::from_bindle(bindle, server, working_dir).await?, _ => bail!("Loading from a bindle requires a Bindle server URL"), }, (Some(_), Some(_)) => bail!("Specify only one of app file or bindle ID"), }; - let manifest_url = match app.info.origin { - spin_manifest::ApplicationOrigin::File(path) => { - format!("file:{}", path.canonicalize()?.to_string_lossy()) - } - spin_manifest::ApplicationOrigin::Bindle { id, server } => { - format!("bindle+{}?id={}", server, id) + // Apply --env to component environments + if !self.env.is_empty() { + for component in app.components.iter_mut() { + component.wasm.environment.extend(self.env.iter().cloned()); } - }; + } let trigger_type = match app.info.trigger { ApplicationTrigger::Http(_) => "http", ApplicationTrigger::Redis(_) => "redis", }; + // Build and write app lock file + let locked_app = spin_trigger::locked::build_locked_app(app, working_dir)?; + let locked_path = working_dir.join("spin.lock"); + let locked_app_contents = + serde_json::to_vec_pretty(&locked_app).context("failed to serialize locked app")?; + std::fs::write(&locked_path, locked_app_contents) + .with_context(|| format!("failed to write {:?}", locked_path))?; + let locked_url = format!("file://{}", locked_path.to_string_lossy()); + + // For `spin up --help`, we just want the executor to dump its own argument usage info let trigger_args = if self.help { vec![OsString::from("--help-args-only")] } else { @@ -170,24 +167,16 @@ impl UpCommand { // via hard-link. I think it should be fine as long as we aren't `setuid`ing this binary. let mut cmd = std::process::Command::new(std::env::current_exe().unwrap()); cmd.arg("trigger") - .env("SPIN_WORKING_DIR", working_dir) - .env("SPIN_MANIFEST_URL", manifest_url) - .env("SPIN_TRIGGER_TYPE", trigger_type) - .env( - "SPIN_ALLOW_TRANSIENT_WRITE", - self.allow_transient_write.to_string(), - ) + .env(SPIN_WORKING_DIR, working_dir) + .env(SPIN_LOCKED_URL, locked_url) .arg(trigger_type) .args(trigger_args); - if let Some(bindle_server) = self.server { - cmd.env(BINDLE_URL_ENV, bindle_server); - } - tracing::trace!("Running trigger executor: {:?}", cmd); let mut child = cmd.spawn().context("Failed to execute trigger")?; + // Terminate trigger executor if `spin up` itself receives a termination signal #[cfg(not(windows))] { // https://github.com/nix-rust/nix/issues/656 @@ -232,3 +221,12 @@ impl WorkingDirectory { } } } + +// Parse the environment variables passed in `key=value` pairs. +fn parse_env_var(s: &str) -> Result<(String, String)> { + let parts: Vec<_> = s.splitn(2, '=').collect(); + if parts.len() != 2 { + bail!("Environment variable must be of the form `key=value`"); + } + Ok((parts[0].to_owned(), parts[1].to_owned())) +} From db226364f3e911e73d1cc3cc43f6a3e1c1090a31 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 12:00:20 -0400 Subject: [PATCH 14/28] Remove spin-engine crate Signed-off-by: Lann Martin --- Cargo.lock | 215 ++++----------- Cargo.toml | 2 - crates/engine/Cargo.toml | 25 -- crates/engine/src/host_component.rs | 100 ------- crates/engine/src/io.rs | 256 ------------------ crates/engine/src/lib.rs | 399 ---------------------------- docs/content/architecture.md | 26 +- tests/integration.rs | 4 +- 8 files changed, 49 insertions(+), 978 deletions(-) delete mode 100644 crates/engine/Cargo.toml delete mode 100644 crates/engine/src/host_component.rs delete mode 100644 crates/engine/src/io.rs delete mode 100644 crates/engine/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index 89053d14a..bc2b5d589 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -280,31 +280,12 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04e142bbbe9d5d6a2dd0387f887a000b41f4c82fb1226316dfb4cc8dbc3b1a29" dependencies = [ - "cap-primitives 0.25.2", - "cap-std 0.25.2", - "io-lifetimes 0.7.3", + "cap-primitives", + "cap-std", + "io-lifetimes", "windows-sys", ] -[[package]] -name = "cap-primitives" -version = "0.24.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb8fca3e81fae1d91a36e9784ca22a39ef623702b5f7904d89dc31f10184a178" -dependencies = [ - "ambient-authority", - "errno", - "fs-set-times 0.15.0", - "io-extras 0.13.2", - "io-lifetimes 0.5.3", - "ipnet", - "maybe-owned", - "rustix 0.33.7", - "winapi", - "winapi-util", - "winx 0.31.0", -] - [[package]] name = "cap-primitives" version = "0.25.2" @@ -313,15 +294,15 @@ checksum = "7f22f4975282dd4f2330ee004f001c4e22f420da9fb474ea600e9af330f1e548" dependencies = [ "ambient-authority", "errno", - "fs-set-times 0.17.1", - "io-extras 0.15.0", - "io-lifetimes 0.7.3", + "fs-set-times", + "io-extras", + "io-lifetimes", "ipnet", "maybe-owned", - "rustix 0.35.9", + "rustix", "winapi-util", "windows-sys", - "winx 0.33.0", + "winx", ] [[package]] @@ -334,30 +315,17 @@ dependencies = [ "rand 0.8.5", ] -[[package]] -name = "cap-std" -version = "0.24.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2247568946095c7765ad2b441a56caffc08027734c634a6d5edda648f04e32eb" -dependencies = [ - "cap-primitives 0.24.4", - "io-extras 0.13.2", - "io-lifetimes 0.5.3", - "ipnet", - "rustix 0.33.7", -] - [[package]] name = "cap-std" version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95624bb0abba6b6ff6fad2e02a7d3945d093d064ac5a3477a308c29fbe3bfd49" dependencies = [ - "cap-primitives 0.25.2", - "io-extras 0.15.0", - "io-lifetimes 0.7.3", + "cap-primitives", + "io-extras", + "io-lifetimes", "ipnet", - "rustix 0.35.9", + "rustix", ] [[package]] @@ -366,10 +334,10 @@ version = "0.25.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46a2d284862edf6e431e9ad4e109c02855157904cebaceae6f042b124a1a21e2" dependencies = [ - "cap-primitives 0.25.2", + "cap-primitives", "once_cell", - "rustix 0.35.9", - "winx 0.33.0", + "rustix", + "winx", ] [[package]] @@ -1119,25 +1087,14 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fs-set-times" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7df62ee66ee2d532ea8d567b5a3f0d03ecd64636b98bad5be1e93dcc918b92aa" -dependencies = [ - "io-lifetimes 0.5.3", - "rustix 0.33.7", - "winapi", -] - [[package]] name = "fs-set-times" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a267b6a9304912e018610d53fe07115d8b530b160e85db4d2d3a59f3ddde1aec" dependencies = [ - "io-lifetimes 0.7.3", - "rustix 0.35.9", + "io-lifetimes", + "rustix", "windows-sys", ] @@ -1601,32 +1558,16 @@ dependencies = [ "cfg-if", ] -[[package]] -name = "io-extras" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d0c937cc9891c12eaa8c63ad347e4a288364b1328b924886970b47a14ab8f8f8" -dependencies = [ - "io-lifetimes 0.5.3", - "winapi", -] - [[package]] name = "io-extras" version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5d8c2ab5becd8720e30fd25f8fa5500d8dc3fceadd8378f05859bd7b46fc49" dependencies = [ - "io-lifetimes 0.7.3", + "io-lifetimes", "windows-sys", ] -[[package]] -name = "io-lifetimes" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec58677acfea8a15352d42fc87d11d63596ade9239e0a7c9352914417515dbe6" - [[package]] name = "io-lifetimes" version = "0.7.3" @@ -1650,8 +1591,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0d508111813f9af3afd2f92758f77e4ed2cc9371b642112c6a48d22eb73105c5" dependencies = [ "hermit-abi 0.2.5", - "io-lifetimes 0.7.3", - "rustix 0.35.9", + "io-lifetimes", + "rustix", "windows-sys", ] @@ -1768,12 +1709,6 @@ dependencies = [ "vcpkg", ] -[[package]] -name = "linux-raw-sys" -version = "0.0.42" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5284f00d480e1c39af34e72f8ad60b94f47007e3481cd3b731c1d67190ddc7b7" - [[package]] name = "linux-raw-sys" version = "0.0.46" @@ -1917,7 +1852,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "480b5a5de855d11ff13195950bdc8b98b5e942ef47afc447f6615cdcc4e15d80" dependencies = [ - "rustix 0.35.9", + "rustix", ] [[package]] @@ -2841,22 +2776,6 @@ version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" -[[package]] -name = "rustix" -version = "0.33.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938a344304321a9da4973b9ff4f9f8db9caf4597dfd9dda6a60b523340a0fff0" -dependencies = [ - "bitflags", - "errno", - "io-lifetimes 0.5.3", - "itoa 1.0.3", - "libc", - "linux-raw-sys 0.0.42", - "once_cell", - "winapi", -] - [[package]] name = "rustix" version = "0.35.9" @@ -2865,10 +2784,10 @@ checksum = "72c825b8aa8010eb9ee99b75f05e10180b9278d161583034d7574c9d617aeada" dependencies = [ "bitflags", "errno", - "io-lifetimes 0.7.3", + "io-lifetimes", "itoa 1.0.3", "libc", - "linux-raw-sys 0.0.46", + "linux-raw-sys", "once_cell", "windows-sys", ] @@ -2924,16 +2843,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "sanitize-filename" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf18934a12018228c5b55a6dae9df5d0641e3566b3630cb46cc55564068e7c2f" -dependencies = [ - "lazy_static", - "regex", -] - [[package]] name = "sanitize-filename" version = "0.4.0" @@ -3337,7 +3246,6 @@ dependencies = [ "sha2 0.10.3", "spin-build", "spin-config", - "spin-engine", "spin-http", "spin-loader", "spin-manifest", @@ -3387,26 +3295,6 @@ dependencies = [ "wasmtime-wasi", ] -[[package]] -name = "spin-engine" -version = "0.2.0" -dependencies = [ - "anyhow", - "bytes", - "cap-std 0.24.4", - "dirs 4.0.0", - "sanitize-filename 0.3.0", - "spin-manifest", - "tempfile", - "tokio", - "tracing", - "wasi-cap-std-sync", - "wasi-common", - "wasmtime", - "wasmtime-wasi", - "wit-bindgen-wasmtime", -] - [[package]] name = "spin-http" version = "0.2.0" @@ -3641,7 +3529,7 @@ dependencies = [ "outbound-http", "outbound-pg", "outbound-redis", - "sanitize-filename 0.4.0", + "sanitize-filename", "serde", "serde_json", "spin-app", @@ -3752,11 +3640,11 @@ dependencies = [ "atty", "bitflags", "cap-fs-ext", - "cap-std 0.25.2", - "io-lifetimes 0.7.3", - "rustix 0.35.9", + "cap-std", + "io-lifetimes", + "rustix", "windows-sys", - "winx 0.33.0", + "winx", ] [[package]] @@ -4318,14 +4206,14 @@ dependencies = [ "async-trait", "cap-fs-ext", "cap-rand", - "cap-std 0.25.2", + "cap-std", "cap-time-ext", - "fs-set-times 0.17.1", - "io-extras 0.15.0", - "io-lifetimes 0.7.3", + "fs-set-times", + "io-extras", + "io-lifetimes", "is-terminal", "lazy_static", - "rustix 0.35.9", + "rustix", "system-interface", "tracing", "wasi-common", @@ -4341,9 +4229,9 @@ dependencies = [ "anyhow", "bitflags", "cap-rand", - "cap-std 0.25.2", - "io-extras 0.15.0", - "rustix 0.35.9", + "cap-std", + "io-extras", + "rustix", "thiserror", "tracing", "wiggle", @@ -4357,11 +4245,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2ab325bba31ae9286b8ebdc18d32a43d6471312c9bc4e477240be444e00ec4f4" dependencies = [ "anyhow", - "cap-std 0.25.2", - "io-extras 0.15.0", - "io-lifetimes 0.7.3", + "cap-std", + "io-extras", + "io-lifetimes", "lazy_static", - "rustix 0.35.9", + "rustix", "tokio", "wasi-cap-std-sync", "wasi-common", @@ -4498,7 +4386,7 @@ dependencies = [ "directories-next", "file-per-thread-logger", "log", - "rustix 0.35.9", + "rustix", "serde", "sha2 0.9.9", "toml", @@ -4556,7 +4444,7 @@ checksum = "2f6aba0b317746e8213d1f36a4c51974e66e69c1f05bfc09ed29b4d4bda290eb" dependencies = [ "cc", "cfg-if", - "rustix 0.35.9", + "rustix", "windows-sys", ] @@ -4577,7 +4465,7 @@ dependencies = [ "object 0.28.4", "region", "rustc-demangle", - "rustix 0.35.9", + "rustix", "serde", "target-lexicon", "thiserror", @@ -4595,7 +4483,7 @@ checksum = "55e23273fddce8cab149a0743c46932bf4910268641397ed86b46854b089f38f" dependencies = [ "lazy_static", "object 0.28.4", - "rustix 0.35.9", + "rustix", ] [[package]] @@ -4617,7 +4505,7 @@ dependencies = [ "more-asserts", "rand 0.8.5", "region", - "rustix 0.35.9", + "rustix", "thiserror", "wasmtime-environ", "wasmtime-fiber", @@ -4847,17 +4735,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "winx" -version = "0.31.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d5973cb8cd94a77d03ad7e23bbe14889cb29805da1cec0e4aff75e21aebded" -dependencies = [ - "bitflags", - "io-lifetimes 0.5.3", - "winapi", -] - [[package]] name = "winx" version = "0.33.0" @@ -4865,7 +4742,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7b01e010390eb263a4518c8cebf86cb67469d1511c00b749a47b64c39e8054d" dependencies = [ "bitflags", - "io-lifetimes 0.7.3", + "io-lifetimes", "windows-sys", ] diff --git a/Cargo.toml b/Cargo.toml index afd00c2c8..40d9cc61d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,6 @@ serde_json = "1.0.82" sha2 = "0.10.2" spin-build = { path = "crates/build" } spin-config = { path = "crates/config" } -spin-engine = { path = "crates/engine" } spin-http = { path = "crates/http" } spin-loader = { path = "crates/loader" } spin-manifest = { path = "crates/manifest" } @@ -75,7 +74,6 @@ members = [ "crates/build", "crates/config", "crates/core", - "crates/engine", "crates/http", "crates/loader", "crates/manifest", diff --git a/crates/engine/Cargo.toml b/crates/engine/Cargo.toml deleted file mode 100644 index 99f1d4784..000000000 --- a/crates/engine/Cargo.toml +++ /dev/null @@ -1,25 +0,0 @@ -[package] -name = "spin-engine" -version = "0.2.0" -edition = "2021" -authors = ["Fermyon Engineering "] - -[dependencies] -anyhow = "1.0.44" -bytes = "1.1.0" -dirs = "4.0" -sanitize-filename = "0.3.0" -spin-manifest = { path = "../manifest" } -tempfile = "3.3.0" -tokio = { version = "1.10.0", features = [ "full" ] } -tracing = { version = "0.1", features = [ "log" ] } -wasi-cap-std-sync = "0.39.1" -wasi-common = "0.39.1" -wasmtime = { version = "0.39.1", features = [ "async" ] } -wasmtime-wasi = "0.39.1" -cap-std = "0.24.1" - -[dev-dependencies.wit-bindgen-wasmtime] -git = "https://github.com/bytecodealliance/wit-bindgen" -rev = "cb871cfa1ee460b51eb1d144b175b9aab9c50aba" -features = [ "async" ] diff --git a/crates/engine/src/host_component.rs b/crates/engine/src/host_component.rs deleted file mode 100644 index 059fdeaad..000000000 --- a/crates/engine/src/host_component.rs +++ /dev/null @@ -1,100 +0,0 @@ -use std::{any::Any, marker::PhantomData}; - -use anyhow::Result; -use spin_manifest::CoreComponent; -use wasmtime::Linker; - -use crate::RuntimeContext; - -/// Represents a host implementation of a Wasm interface. -pub trait HostComponent: Send + Sync { - /// Host component runtime state. - type State: Any + Send; - - /// Add this component to the given Linker, using the given runtime state-getting handle. - fn add_to_linker( - linker: &mut Linker>, - state_handle: HostComponentsStateHandle, - ) -> Result<()>; - - /// Build a new runtime state object for the given component. - fn build_state(&self, component: &CoreComponent) -> Result; -} -type HostComponentState = Box; - -type StateBuilder = Box Result + Send + Sync>; - -#[derive(Default)] -pub(crate) struct HostComponents { - state_builders: Vec, -} - -impl HostComponents { - pub(crate) fn insert( - &mut self, - linker: &mut Linker>, - host_component: Component, - ) -> Result<()> { - let handle = HostComponentsStateHandle { - idx: self.state_builders.len(), - _phantom: PhantomData, - }; - Component::add_to_linker(linker, handle)?; - self.state_builders.push(Box::new(move |c| { - Ok(Box::new(host_component.build_state(c)?)) - })); - Ok(()) - } - - pub(crate) fn build_state(&self, c: &CoreComponent) -> Result { - Ok(HostComponentsState( - self.state_builders - .iter() - .map(|build_state| build_state(c)) - .collect::>()?, - )) - } -} - -/// A collection of host components state. -#[derive(Default)] -pub struct HostComponentsState(Vec); - -/// A handle to component state, used in HostComponent::add_to_linker. -pub struct HostComponentsStateHandle { - idx: usize, - _phantom: PhantomData T>, -} - -impl HostComponentsStateHandle { - /// Get a ref to the component state associated with this handle from the RuntimeContext. - pub fn get<'a, U>(&self, ctx: &'a RuntimeContext) -> &'a T { - ctx.host_components_state - .0 - .get(self.idx) - .unwrap() - .downcast_ref() - .unwrap() - } - - /// Get a mutable ref to the component state associated with this handle from the RuntimeContext. - pub fn get_mut<'a, U>(&self, ctx: &'a mut RuntimeContext) -> &'a mut T { - ctx.host_components_state - .0 - .get_mut(self.idx) - .unwrap() - .downcast_mut() - .unwrap() - } -} - -impl Clone for HostComponentsStateHandle { - fn clone(&self) -> Self { - Self { - idx: self.idx, - _phantom: PhantomData, - } - } -} - -impl Copy for HostComponentsStateHandle {} diff --git a/crates/engine/src/io.rs b/crates/engine/src/io.rs deleted file mode 100644 index 6553fc9c0..000000000 --- a/crates/engine/src/io.rs +++ /dev/null @@ -1,256 +0,0 @@ -use std::{ - collections::HashSet, - fmt::Debug, - io::{LineWriter, Write}, - sync::{Arc, RwLock, RwLockReadGuard}, -}; - -use wasi_common::{ - pipe::{ReadPipe, WritePipe}, - WasiFile, -}; - -/// Prepares a WASI pipe which writes to a memory buffer, optionally -/// copying to the specified output stream. -pub fn redirect_to_mem_buffer( - follow: Follow, -) -> (WritePipe, Arc>) { - let immediate = follow.writer(); - - let buffer: Vec = vec![]; - let std_dests = WriteDestinations { buffer, immediate }; - let lock = Arc::new(RwLock::new(std_dests)); - let std_pipe = WritePipe::from_shared(lock.clone()); - - (std_pipe, lock) -} - -/// Which components should have their logs followed on stdout/stderr. -#[derive(Clone, Debug)] -pub enum FollowComponents { - /// No components should have their logs followed. - None, - /// Only the specified components should have their logs followed. - Named(HashSet), - /// All components should have their logs followed. - All, -} - -impl FollowComponents { - /// Whether a given component should have its logs followed on stdout/stderr. - pub fn should_follow(&self, component_id: &str) -> bool { - match self { - Self::None => false, - Self::All => true, - Self::Named(ids) => ids.contains(component_id), - } - } -} - -impl Default for FollowComponents { - fn default() -> Self { - Self::None - } -} - -/// The buffers in which Wasm module output has been saved. -pub trait OutputBuffers { - /// The buffer in which stdout has been saved. - fn stdout(&self) -> &[u8]; - /// The buffer in which stderr has been saved. - fn stderr(&self) -> &[u8]; -} - -/// A set of redirected standard I/O streams with which -/// a Wasm module is to be run. -pub struct ModuleIoRedirects { - /// pipes for ModuleIoRedirects - pub pipes: RedirectPipes, - /// read handles for ModuleIoRedirects - pub read_handles: RedirectReadHandles, -} - -impl Default for ModuleIoRedirects { - fn default() -> Self { - Self::new(false) - } -} - -impl ModuleIoRedirects { - /// Constructs the ModuleIoRedirects, and RedirectReadHandles instances the default way - pub fn new(follow: bool) -> Self { - let rrh = RedirectReadHandles::new(follow); - - let in_stdpipe: Box = Box::new(ReadPipe::from(vec![])); - let out_stdpipe: Box = Box::new(WritePipe::from_shared(rrh.stdout.clone())); - let err_stdpipe: Box = Box::new(WritePipe::from_shared(rrh.stderr.clone())); - - Self { - pipes: RedirectPipes { - stdin: in_stdpipe, - stdout: out_stdpipe, - stderr: err_stdpipe, - }, - read_handles: rrh, - } - } -} - -/// Pipes from `ModuleIoRedirects` -pub struct RedirectPipes { - pub(crate) stdin: Box, - pub(crate) stdout: Box, - pub(crate) stderr: Box, -} - -impl RedirectPipes { - /// Constructs an instance from a set of WasiFile objects. - pub fn new( - stdin: Box, - stdout: Box, - stderr: Box, - ) -> Self { - Self { - stdin, - stdout, - stderr, - } - } -} - -/// The destinations to which redirected module output will be written. -/// Used for subsequently reading back the output. -pub struct RedirectReadHandles { - stdout: Arc>, - stderr: Arc>, -} - -impl Default for RedirectReadHandles { - fn default() -> Self { - Self::new(false) - } -} - -impl RedirectReadHandles { - /// Creates a new RedirectReadHandles instance - pub fn new(follow: bool) -> Self { - let out_immediate = Follow::stdout(follow).writer(); - let err_immediate = Follow::stderr(follow).writer(); - - let out_buffer: Vec = vec![]; - let err_buffer: Vec = vec![]; - - let out_std_dests = WriteDestinations { - buffer: out_buffer, - immediate: out_immediate, - }; - let err_std_dests = WriteDestinations { - buffer: err_buffer, - immediate: err_immediate, - }; - - Self { - stdout: Arc::new(RwLock::new(out_std_dests)), - stderr: Arc::new(RwLock::new(err_std_dests)), - } - } - /// Acquires a read lock for the in-memory output buffers. - pub fn read(&self) -> impl OutputBuffers + '_ { - RedirectReadHandlesLock { - stdout: self.stdout.read().unwrap(), - stderr: self.stderr.read().unwrap(), - } - } -} - -struct RedirectReadHandlesLock<'a> { - stdout: RwLockReadGuard<'a, WriteDestinations>, - stderr: RwLockReadGuard<'a, WriteDestinations>, -} - -impl<'a> OutputBuffers for RedirectReadHandlesLock<'a> { - fn stdout(&self) -> &[u8] { - self.stdout.buffer() - } - fn stderr(&self) -> &[u8] { - self.stderr.buffer() - } -} - -/// Indicates whether a memory redirect should also pipe the output to -/// the console so it can be followed live. -pub enum Follow { - /// Do not pipe to console - only write to memory. - None, - /// Also pipe to stdout. - Stdout, - /// Also pipe to stderr. - Stderr, -} - -impl Follow { - pub(crate) fn writer(&self) -> Box { - match self { - Self::None => Box::new(DiscardingWriter), - Self::Stdout => Box::new(LineWriter::new(std::io::stdout())), - Self::Stderr => Box::new(LineWriter::new(std::io::stderr())), - } - } - - /// Follow on stdout if so specified. - pub fn stdout(follow_on_stdout: bool) -> Self { - if follow_on_stdout { - Self::Stdout - } else { - Self::None - } - } - - /// Follow on stderr if so specified. - pub fn stderr(follow_on_stderr: bool) -> Self { - if follow_on_stderr { - Self::Stderr - } else { - Self::None - } - } -} - -/// The destinations to which a component writes an output stream. -pub struct WriteDestinations { - buffer: Vec, - immediate: Box, -} - -impl WriteDestinations { - /// The memory buffer to which a component writes an output stream. - pub fn buffer(&self) -> &[u8] { - &self.buffer - } -} - -impl Write for WriteDestinations { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - let written = self.buffer.write(buf)?; - self.immediate.write_all(&buf[0..written])?; - Ok(written) - } - - fn flush(&mut self) -> std::io::Result<()> { - self.buffer.flush()?; - self.immediate.flush()?; - Ok(()) - } -} - -struct DiscardingWriter; - -impl Write for DiscardingWriter { - fn write(&mut self, buf: &[u8]) -> std::io::Result { - Ok(buf.len()) - } - - fn flush(&mut self) -> std::io::Result<()> { - Ok(()) - } -} diff --git a/crates/engine/src/lib.rs b/crates/engine/src/lib.rs deleted file mode 100644 index ffbd803a1..000000000 --- a/crates/engine/src/lib.rs +++ /dev/null @@ -1,399 +0,0 @@ -//! A Spin execution context for applications. - -#![deny(missing_docs)] - -/// Host components. -pub mod host_component; -/// Input / Output redirects. -pub mod io; - -use std::{collections::HashMap, io::Write, path::PathBuf, sync::Arc}; - -use anyhow::{bail, Context, Result}; -use host_component::{HostComponent, HostComponents, HostComponentsState}; -use io::{FollowComponents, OutputBuffers, RedirectPipes}; -use spin_manifest::{CoreComponent, DirectoryMount, ModuleSource}; -use tokio::{ - task::JoinHandle, - time::{sleep, Duration}, -}; -use tracing::{instrument, log}; -use wasi_common::WasiCtx; -use wasmtime::{Instance, InstancePre, Linker, Module, Store}; -use wasmtime_wasi::{ambient_authority, Dir, WasiCtxBuilder}; - -const SPIN_HOME: &str = ".spin"; - -/// Builder-specific configuration. -#[derive(Clone, Debug, Default)] -pub struct ExecutionContextConfiguration { - /// Component configuration. - pub components: Vec, - /// Label for logging, etc. - pub label: String, - /// Log directory on host. - pub log_dir: Option, - /// Component log following configuration. - pub follow_components: FollowComponents, -} - -/// Top-level runtime context data to be passed to a component. -#[derive(Default)] -pub struct RuntimeContext { - /// WASI context data. - pub wasi: Option, - /// Host components state. - pub host_components_state: HostComponentsState, - /// Generic runtime data that can be configured by specialized engines. - pub data: Option, -} - -/// The engine struct that encapsulate wasmtime engine -#[derive(Clone, Default)] -pub struct Engine(wasmtime::Engine); - -impl Engine { - /// Create a new engine and initialize it with the given config. - pub fn new(mut config: wasmtime::Config) -> Result { - config.async_support(true); - Ok(Self(wasmtime::Engine::new(&config)?)) - } - - /// Get a clone of the internal `wasmtime::Engine`. - /// WARNING: The configuration of this Engine is likely to change in the future, and - /// will not be covered by any future stability guarantees. - pub fn inner(&self) -> wasmtime::Engine { - self.0.clone() - } -} - -/// An execution context builder. -pub struct Builder { - config: ExecutionContextConfiguration, - linker: Linker>, - store: Store>, - engine: Engine, - host_components: HostComponents, -} - -impl Builder { - /// Creates a new instance of the execution builder. - pub fn new(config: ExecutionContextConfiguration) -> Result> { - Self::with_engine(config, Engine::new(Default::default())?) - } - - /// Creates a new instance of the execution builder with the given wasmtime::Config. - pub fn with_engine( - config: ExecutionContextConfiguration, - engine: Engine, - ) -> Result> { - let data = RuntimeContext::default(); - let linker = Linker::new(&engine.0); - let store = Store::new(&engine.0, data); - let host_components = Default::default(); - - Ok(Self { - config, - linker, - store, - engine, - host_components, - }) - } - - /// Returns the current ExecutionContextConfiguration. - pub fn config(&self) -> &ExecutionContextConfiguration { - &self.config - } - - /// Configures the WASI linker imports for the current execution context. - pub fn link_wasi(&mut self) -> Result<&mut Self> { - wasmtime_wasi::add_to_linker(&mut self.linker, |ctx| ctx.wasi.as_mut().unwrap())?; - Ok(self) - } - - /// Adds a HostComponent to the execution context. - pub fn add_host_component( - &mut self, - host_component: impl HostComponent + 'static, - ) -> Result<&mut Self> { - self.host_components - .insert(&mut self.linker, host_component)?; - Ok(self) - } - - /// Builds a new instance of the execution context. - #[instrument(skip(self))] - pub async fn build(mut self) -> Result> { - let _sloth_warning = warn_if_slothful(); - let mut components = HashMap::new(); - for c in &self.config.components { - let core = c.clone(); - let module = match c.source.clone() { - ModuleSource::FileReference(p) => { - let module = Module::from_file(&self.engine.0, &p).with_context(|| { - format!( - "Cannot create module for component {} from file {}", - &c.id, - &p.display() - ) - })?; - log::trace!("Created module for component {} from file {:?}", &c.id, &p); - module - } - ModuleSource::Buffer(bytes, info) => { - let module = - Module::from_binary(&self.engine.0, &bytes).with_context(|| { - format!("Cannot create module for component {} from {}", &c.id, info) - })?; - log::trace!( - "Created module for component {} from {} with size {}", - &c.id, - info, - bytes.len() - ); - module - } - }; - - let pre = Arc::new(self.linker.instantiate_pre(&mut self.store, &module)?); - log::trace!("Created pre-instance from module for component {}.", &c.id); - - components.insert(c.id.clone(), Component { core, pre }); - } - - log::trace!("Execution context initialized."); - - Ok(ExecutionContext { - config: self.config, - engine: self.engine, - components, - host_components: Arc::new(self.host_components), - }) - } - - /// Configures default host interface implementations. - pub fn link_defaults(&mut self) -> Result<&mut Self> { - self.link_wasi() - } - - /// Builds a new default instance of the execution context. - pub async fn build_default( - config: ExecutionContextConfiguration, - ) -> Result> { - let mut builder = Self::new(config)?; - builder.link_defaults()?; - builder.build().await - } -} - -/// Component for the execution context. -#[derive(Clone)] -pub struct Component { - /// Configuration for the component. - pub core: CoreComponent, - /// The pre-instance of the component - pub pre: Arc>>, -} - -/// A generic execution context for WebAssembly components. -#[derive(Clone)] -pub struct ExecutionContext { - /// Top-level runtime configuration. - pub config: ExecutionContextConfiguration, - /// Wasmtime engine. - pub engine: Engine, - /// Collection of pre-initialized (and already linked) components. - pub components: HashMap>, - - host_components: Arc, -} - -impl ExecutionContext { - /// Creates a store for a given component given its configuration and runtime data. - #[instrument(skip(self, data, io))] - pub async fn prepare_component( - &self, - component: &str, - data: Option, - io: Option, - env: Option>, - args: Option>, - ) -> Result<(Store>, Instance)> { - log::trace!("Preparing component {}", component); - let component = match self.components.get(component) { - Some(c) => c, - None => bail!("Cannot find component {}", component), - }; - - let mut store = self.store(component, data, io, env, args)?; - let instance = component.pre.instantiate_async(&mut store).await?; - - Ok((store, instance)) - } - - /// Save logs for a given component in the log directory on the host - pub fn save_output_to_logs( - &self, - ior: impl OutputBuffers, - component: &str, - save_stdout: bool, - save_stderr: bool, - ) -> Result<()> { - let sanitized_label = sanitize(&self.config.label); - let sanitized_component_name = sanitize(&component); - - let log_dir = match &self.config.log_dir { - Some(l) => l.clone(), - None => match dirs::home_dir() { - Some(h) => h.join(SPIN_HOME).join(&sanitized_label).join("logs"), - None => PathBuf::from(&sanitized_label).join("logs"), - }, - }; - - let stdout_filename = log_dir.join(sanitize(format!( - "{}_{}.txt", - sanitized_component_name, "stdout", - ))); - - let stderr_filename = log_dir.join(sanitize(format!( - "{}_{}.txt", - sanitized_component_name, "stderr" - ))); - - std::fs::create_dir_all(&log_dir)?; - - log::trace!("Saving logs to {:?} {:?}", stdout_filename, stderr_filename); - - if save_stdout { - let mut file = std::fs::OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(stdout_filename)?; - let contents = ior.stdout(); - file.write_all(contents)?; - } - - if save_stderr { - let mut file = std::fs::OpenOptions::new() - .write(true) - .append(true) - .create(true) - .open(stderr_filename)?; - let contents = ior.stderr(); - file.write_all(contents)?; - } - - Ok(()) - } - /// Creates a store for a given component given its configuration and runtime data. - fn store( - &self, - component: &Component, - data: Option, - io: Option, - env: Option>, - args: Option>, - ) -> Result>> { - log::trace!("Creating store."); - let (env, dirs) = Self::wasi_config(component, env)?; - let mut ctx = RuntimeContext::default(); - let mut wasi_ctx = WasiCtxBuilder::new() - .args(&args.unwrap_or_default())? - .envs(&env)?; - match io { - Some(r) => { - wasi_ctx = wasi_ctx.stderr(r.stderr).stdout(r.stdout).stdin(r.stdin); - } - None => wasi_ctx = wasi_ctx.inherit_stdio(), - }; - - for dir in dirs { - let guest = dir.guest; - let host = dir.host; - wasi_ctx = - wasi_ctx.preopened_dir(Dir::open_ambient_dir(host, ambient_authority())?, guest)?; - } - - ctx.host_components_state = self.host_components.build_state(&component.core)?; - - ctx.wasi = Some(wasi_ctx.build()); - ctx.data = data; - - let store = Store::new(&self.engine.0, ctx); - Ok(store) - } - - #[allow(clippy::type_complexity)] - fn wasi_config( - component: &Component, - env: Option>, - ) -> Result<(Vec<(String, String)>, Vec)> { - let mut res = vec![]; - - for (k, v) in &component.core.wasm.environment { - res.push((k.clone(), v.clone())); - } - - // Custom environment variables currently take precedence over component-defined - // environment variables. This might change in the future. - if let Some(envs) = env { - for (k, v) in envs { - res.push((k.clone(), v.clone())); - } - }; - - let dirs = component.core.wasm.mounts.clone(); - - Ok((res, dirs)) - } -} - -fn sanitize(name: impl AsRef) -> String { - // options block copied from sanitize_filename project readme - let options = sanitize_filename::Options { - // true by default, truncates to 255 bytes - truncate: true, - // default value depends on the OS, removes reserved names like `con` from start of strings on Windows - windows: true, - // str to replace sanitized chars/strings - replacement: "", - }; - - // filename logic defined in the project works for directory names as well - // refer to: https://github.com/kardeiz/sanitize-filename/blob/f5158746946ed81015c3a33078dedf164686da19/src/lib.rs#L76-L165 - sanitize_filename::sanitize_with_options(name, options) -} - -const SLOTH_WARNING_DELAY_MILLIS: u64 = 1250; - -struct SlothWarning { - warning: JoinHandle, -} - -impl Drop for SlothWarning { - fn drop(&mut self) { - self.warning.abort() - } -} - -fn warn_if_slothful() -> SlothWarning<()> { - let warning = tokio::spawn(warn_slow()); - SlothWarning { warning } -} - -#[cfg(debug_assertions)] -async fn warn_slow() { - sleep(Duration::from_millis(SLOTH_WARNING_DELAY_MILLIS)).await; - println!("This is a debug build - preparing Wasm modules might take a few seconds"); - println!("If you're experiencing long startup times please switch to the release build"); - println!(); -} - -#[cfg(not(debug_assertions))] -async fn warn_slow() { - sleep(Duration::from_millis(SLOTH_WARNING_DELAY_MILLIS)).await; - println!("Preparing Wasm modules is taking a few seconds..."); - println!(); -} diff --git a/docs/content/architecture.md b/docs/content/architecture.md index 77def7383..bcc24f0dd 100644 --- a/docs/content/architecture.md +++ b/docs/content/architecture.md @@ -34,30 +34,6 @@ application configuration ([#40](https://github.com/fermyon/spin/issues/40) explores a trigger handling multiple applications), starts an HTTP listener, and for each new request, it routes it to the component configured in the application configuration. Then, it instantiates the WebAssembly module (using a -`spin_engine::ExecutionContext`) and uses the appropriate executor (either the +`spin_core::Engine`) and uses the appropriate executor (either the `SpinHttpExecutor` or the `WagiHttpExecutor`, based on the component configuration) to handle the request and return the response. - -## The Spin execution context - -The Spin execution context (or "Spin engine") is the part of Spin that executes -WebAssembly components using the -[Wasmtime](https://github.com/bytecodealliance/wasmtime) WebAssembly runtime. It -is implemented in the `spin-engine` crate, and serves as -the part of Spin that takes a fully formed application configuration and creates -Wasm instances based on the component configurations. - -There are two important concepts in this crate: - -- `spin_engine::Builder` — the builder for creating an execution context. It is - created using an `ExecutionContextConfiguration` object (which contains a Spin - application and Wasmtime configuration), and implements the logic for - configuring WASI and the other host implementations provided by Spin. The - builder exposes the Wasmtime - [`Linker`](https://docs.rs/wasmtime/latest/wasmtime/struct.Linker.html), - [`Engine`](https://docs.rs/wasmtime/latest/wasmtime/struct.Engine.html), and - [`Store>`](https://docs.rs/wasmtime/latest/wasmtime/struct.Store.html) - (where `RuntimeContext` is the internal Spin context, which is detailed - later in the document), and it uses them to [pre-instantiate]() - -- `spin_engine::ExecutionContext` — the main execution engine in Spin. diff --git a/tests/integration.rs b/tests/integration.rs index 8b0972304..07d1fe3de 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -688,7 +688,7 @@ mod integration_tests { .args(args) .env( "RUST_LOG", - "spin=trace,spin_loader=trace,spin_engine=trace,spin_http=trace", + "spin=trace,spin_loader=trace,spin_core=trace,spin_http=trace", ) .spawn() .with_context(|| "executing Spin")?; @@ -725,7 +725,7 @@ mod integration_tests { .args(args) .env( "RUST_LOG", - "spin=trace,spin_loader=trace,spin_engine=trace,spin_http=trace", + "spin=trace,spin_loader=trace,spin_core=trace,spin_http=trace", ) .spawn() .with_context(|| "executing Spin")?; From be8d94dec11a65abb8319e4f41cb04d38b46fae8 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 16:12:41 -0400 Subject: [PATCH 15/28] Canonicalize file paths in spin-trigger File URIs are supposed to be absolute. Signed-off-by: Lann Martin --- crates/trigger/src/locked.rs | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/crates/trigger/src/locked.rs b/crates/trigger/src/locked.rs index bd7e622bc..6d86637b1 100644 --- a/crates/trigger/src/locked.rs +++ b/crates/trigger/src/locked.rs @@ -20,7 +20,7 @@ const WASM_CONTENT_TYPE: &str = "application/wasm"; /// Construct a LockedApp from the given Application. Any buffered component /// sources will be written to the given `working_dir`. pub fn build_locked_app(app: Application, working_dir: impl Into) -> Result { - let working_dir = working_dir.into(); + let working_dir = working_dir.into().canonicalize()?; LockedAppBuilder { working_dir }.build(app) } @@ -51,7 +51,7 @@ impl LockedAppBuilder { builder.string( "origin", match info.origin { - spin_manifest::ApplicationOrigin::File(path) => file_uri(&path, false)?, + spin_manifest::ApplicationOrigin::File(path) => file_uri(&path)?, spin_manifest::ApplicationOrigin::Bindle { id, server } => { format!("bindle+{server}?id={id}") } @@ -145,7 +145,7 @@ impl LockedAppBuilder { }; LockedComponentSource { content_type: WASM_CONTENT_TYPE.into(), - content: content_ref_path(&path, false)?, + content: content_ref_path(&path)?, } }; @@ -157,7 +157,7 @@ impl LockedAppBuilder { .into_iter() .map(|mount| { Ok(ContentPath { - content: content_ref_path(&mount.host, true)?, + content: content_ref_path(&mount.host)?, path: mount.guest.into(), }) }) @@ -176,20 +176,21 @@ impl LockedAppBuilder { } } -fn content_ref_path(path: &Path, is_dir: bool) -> Result { +fn content_ref_path(path: &Path) -> Result { Ok(ContentRef { - source: Some(file_uri(path, is_dir)?), + source: Some(file_uri(path)?), ..Default::default() }) } -fn file_uri(path: &Path, is_dir: bool) -> Result { - let uri = if is_dir { - url::Url::from_directory_path(path) +fn file_uri(path: &Path) -> Result { + let path = path.canonicalize()?; + let uri = if path.is_dir() { + url::Url::from_directory_path(&path) } else { - url::Url::from_file_path(path) + url::Url::from_file_path(&path) } - .map_err(|err| anyhow!("Could not construct file URI: {err:?}"))?; + .map_err(|_| anyhow!("Could not construct file URI for {path:?}"))?; Ok(uri.to_string()) } @@ -211,7 +212,7 @@ mod tests { [[component]] id = "test-component" source = "test-source.wasm" - files = ["content/*"] + files = ["static.txt"] allowed_http_hosts = ["example.com"] [component.config] test_config = "{{test_var}}" @@ -220,7 +221,7 @@ mod tests { [[component]] id = "test-component-2" - source = "test-source-2.wasm" + source = "test-source.wasm" allowed_http_hosts = ["insecure:allow-all"] [component.trigger] route = "/other" @@ -230,6 +231,8 @@ mod tests { let tempdir = TempDir::new().expect("tempdir"); std::env::set_current_dir(tempdir.path()).unwrap(); std::fs::write("spin.toml", TEST_MANIFEST).expect("write manifest"); + std::fs::write("test-source.wasm", "not actual wasm").expect("write source"); + std::fs::write("static.txt", "content").expect("write static"); let app = spin_loader::local::from_file("spin.toml", &tempdir, &None) .await .expect("load app"); From b0edd46029a052c9121b70be054a7c14f29ec3a0 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 16:15:25 -0400 Subject: [PATCH 16/28] Print failed response body in integration tests Signed-off-by: Lann Martin --- tests/integration.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/integration.rs b/tests/integration.rs index 07d1fe3de..10d54861a 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -647,7 +647,11 @@ mod integration_tests { expected: u16, ) -> Result<()> { let res = req(s, absolute_uri).await?; - assert_eq!(res.status(), expected); + let status = res.status(); + let body = hyper::body::to_bytes(res.into_body()) + .await + .expect("read body"); + assert_eq!(status, expected, "{}", String::from_utf8_lossy(&body)); Ok(()) } From f9a8219966f5b6ca2e52d6b9f68e6a92fb139efc Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 17:11:23 -0400 Subject: [PATCH 17/28] Improve error reporting in spin_app::Error The `:#` formatting modifier causes anyhow::Error to print the full list of causes / contexts. Signed-off-by: Lann Martin --- crates/app/src/lib.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index f0bc26b4e..6484165d7 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -263,11 +263,11 @@ pub type Result = std::result::Result; #[derive(Debug, thiserror::Error)] pub enum Error { - #[error("spin core error: {0}")] + #[error("spin core error: {0:#}")] CoreError(anyhow::Error), - #[error("host component error: {0}")] + #[error("host component error: {0:#}")] HostComponentError(anyhow::Error), - #[error("loader error: {0}")] + #[error("loader error: {0:#}")] LoaderError(anyhow::Error), #[error("manifest error: {0}")] ManifestError(String), From b1873d4a32d44e7b8c92dd7f9d89c39af5301ae2 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 13 Sep 2022 17:38:13 -0400 Subject: [PATCH 18/28] Improve TriggerLoader::load_module error Signed-off-by: Lann Martin --- crates/trigger/src/loader.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/trigger/src/loader.rs b/crates/trigger/src/loader.rs index 9ae915290..bce6f4097 100644 --- a/crates/trigger/src/loader.rs +++ b/crates/trigger/src/loader.rs @@ -47,6 +47,7 @@ impl Loader for TriggerLoader { .context("LockedComponentSource missing source field")?; let path = unwrap_file_uri(source)?; spin_core::Module::from_file(engine, path) + .with_context(|| format!("loading module {path:?}")) } async fn mount_files( From a9f7f5f227187f06c1593a2a45fcee7f76a8a9f8 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 14 Sep 2022 09:11:38 -0400 Subject: [PATCH 19/28] Use 'url' crate to manage file URLs Treating them as strings works fine for *nix OSes, but breaks for Windows. Also, standardize on "URL" vs "URI" in spin-trigger crate. Signed-off-by: Lann Martin --- crates/trigger/src/lib.rs | 17 +++++++++++++---- crates/trigger/src/loader.rs | 23 +++++++++-------------- crates/trigger/src/locked.rs | 8 ++++---- src/commands/up.rs | 7 +++++-- 4 files changed, 31 insertions(+), 24 deletions(-) diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 979fc66bb..94312eeed 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -9,7 +9,7 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::{Context, Result}; +use anyhow::{anyhow, Context, Result}; pub use async_trait::async_trait; use serde::de::DeserializeOwned; @@ -130,9 +130,11 @@ impl TriggerExecutorBuilder { pub fn default_config_providers(&self, app_uri: &str) -> Vec> { // EnvProvider - let dotenv_path = app_uri - .strip_prefix("file://") - .and_then(|path| Path::new(path).parent()) + // Look for a .env file in either the manifest parent directory for local apps + // or the current directory for remote (e.g. bindle) apps. + let dotenv_path = parse_file_url(app_uri) + .as_deref() + .ok() .unwrap_or_else(|| Path::new(".")) .join(".env"); vec![Box::new(EnvProvider::new( @@ -280,3 +282,10 @@ impl TriggerAppEngine { Ok((instance, store)) } } + +pub(crate) fn parse_file_url(url: &str) -> Result { + url::Url::parse(url) + .with_context(|| format!("Invalid URL: {url:?}"))? + .to_file_path() + .map_err(|_| anyhow!("Invalid file URL path: {url:?}")) +} diff --git a/crates/trigger/src/loader.rs b/crates/trigger/src/loader.rs index bce6f4097..4934d18c4 100644 --- a/crates/trigger/src/loader.rs +++ b/crates/trigger/src/loader.rs @@ -1,6 +1,6 @@ #![allow(dead_code)] // Refactor WIP -use std::path::{Path, PathBuf}; +use std::path::PathBuf; use anyhow::{ensure, Context, Result}; use async_trait::async_trait; @@ -10,6 +10,8 @@ use spin_app::{ }; use spin_core::StoreBuilder; +use crate::parse_file_url; + pub struct TriggerLoader { working_dir: PathBuf, allow_transient_write: bool, @@ -26,10 +28,10 @@ impl TriggerLoader { #[async_trait] impl Loader for TriggerLoader { - async fn load_app(&self, uri: &str) -> Result { - let path = unwrap_file_uri(uri)?; + async fn load_app(&self, url: &str) -> Result { + let path = parse_file_url(url)?; let contents = - std::fs::read(path).with_context(|| format!("failed to read manifest at {path:?}"))?; + std::fs::read(&path).with_context(|| format!("failed to read manifest at {path:?}"))?; let app = serde_json::from_slice(&contents).context("failed to parse app lock file JSON")?; Ok(app) @@ -45,8 +47,8 @@ impl Loader for TriggerLoader { .source .as_ref() .context("LockedComponentSource missing source field")?; - let path = unwrap_file_uri(source)?; - spin_core::Module::from_file(engine, path) + let path = parse_file_url(source)?; + spin_core::Module::from_file(engine, &path) .with_context(|| format!("loading module {path:?}")) } @@ -61,7 +63,7 @@ impl Loader for TriggerLoader { .source .as_deref() .with_context(|| format!("Missing 'source' on files mount {content_dir:?}"))?; - let source_path = self.working_dir.join(unwrap_file_uri(source_uri)?); + let source_path = self.working_dir.join(parse_file_url(source_uri)?); ensure!( source_path.is_dir(), "TriggerLoader only supports directory mounts; {source_path:?} is not a directory" @@ -76,10 +78,3 @@ impl Loader for TriggerLoader { Ok(()) } } - -fn unwrap_file_uri(uri: &str) -> Result<&Path> { - Ok(Path::new( - uri.strip_prefix("file://") - .context("TriggerLoader supports only file:// URIs")?, - )) -} diff --git a/crates/trigger/src/locked.rs b/crates/trigger/src/locked.rs index 6d86637b1..0b4bebe1c 100644 --- a/crates/trigger/src/locked.rs +++ b/crates/trigger/src/locked.rs @@ -47,7 +47,7 @@ impl LockedAppBuilder { .string("version", &info.version) .string_option("description", info.description.as_deref()) .serializable("trigger", info.trigger)?; - // Convert ApplicationOrigin to a URI + // Convert ApplicationOrigin to a URL builder.string( "origin", match info.origin { @@ -185,13 +185,13 @@ fn content_ref_path(path: &Path) -> Result { fn file_uri(path: &Path) -> Result { let path = path.canonicalize()?; - let uri = if path.is_dir() { + let url = if path.is_dir() { url::Url::from_directory_path(&path) } else { url::Url::from_file_path(&path) } - .map_err(|_| anyhow!("Could not construct file URI for {path:?}"))?; - Ok(uri.to_string()) + .map_err(|_| anyhow!("Could not construct file URL for {path:?}"))?; + Ok(url.to_string()) } #[cfg(test)] diff --git a/src/commands/up.rs b/src/commands/up.rs index 4e4a6aec6..5ffc32c50 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -4,8 +4,9 @@ use std::{ path::{Path, PathBuf}, }; -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::{CommandFactory, Parser}; +use reqwest::Url; use spin_loader::bindle::BindleConnectionInfo; use spin_manifest::ApplicationTrigger; use spin_trigger::cli::{SPIN_LOCKED_URL, SPIN_WORKING_DIR}; @@ -154,7 +155,9 @@ impl UpCommand { serde_json::to_vec_pretty(&locked_app).context("failed to serialize locked app")?; std::fs::write(&locked_path, locked_app_contents) .with_context(|| format!("failed to write {:?}", locked_path))?; - let locked_url = format!("file://{}", locked_path.to_string_lossy()); + let locked_url = Url::from_file_path(&locked_path) + .map_err(|_| anyhow!("cannot convert to file URL: {locked_path:?}"))? + .to_string(); // For `spin up --help`, we just want the executor to dump its own argument usage info let trigger_args = if self.help { From ce6f0efdac7cba67aa24b9aef7625968e27c25a7 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 14 Sep 2022 13:08:58 -0400 Subject: [PATCH 20/28] Replace OwnedApp Deref with explicit 'borrowed' method The Deref impl used a scary unsafe lifetype cast which was hard to reason about and unsound for use within the spin-app crate. Signed-off-by: Lann Martin --- crates/app/src/lib.rs | 11 +++-------- crates/trigger/src/lib.rs | 11 ++++++----- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 6484165d7..353a6c0bd 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -87,14 +87,9 @@ pub struct OwnedApp { app: App<'this>, } -impl std::ops::Deref for OwnedApp { - type Target = App<'static>; - - fn deref(&self) -> &Self::Target { - unsafe { - // We know that App's lifetime param is for AppLoader, which is owned by self here. - std::mem::transmute::<&App, &App<'static>>(self.borrow_app()) - } +impl OwnedApp { + pub fn borrowed(&self) -> &App { + self.borrow_app() } } diff --git a/crates/trigger/src/lib.rs b/crates/trigger/src/lib.rs index 94312eeed..586e73838 100644 --- a/crates/trigger/src/lib.rs +++ b/crates/trigger/src/lib.rs @@ -110,7 +110,7 @@ impl TriggerExecutorBuilder { }; let app = self.loader.load_owned_app(app_uri).await?; - let app_name = app.require_metadata("name")?; + let app_name = app.borrowed().require_metadata("name")?; let log_dir = { let sanitized_app = sanitize_filename::sanitize(&app_name); @@ -176,6 +176,7 @@ impl TriggerAppEngine { ::TriggerConfig: DeserializeOwned, { let trigger_configs = app + .borrowed() .triggers_with_type(Executor::TRIGGER_TYPE) .map(|trigger| { trigger.typed_config().with_context(|| { @@ -185,7 +186,7 @@ impl TriggerAppEngine { .collect::>()?; let mut component_instance_pres = HashMap::default(); - for component in app.components() { + for component in app.borrowed().components() { let module = component.load_module(&engine).await?; let instance_pre = engine.instantiate_pre(&module)?; component_instance_pres.insert(component.id().to_string(), instance_pre); @@ -204,12 +205,12 @@ impl TriggerAppEngine { /// Returns a reference to the App. pub fn app(&self) -> &App { - &self.app + self.app.borrowed() } /// Returns AppTriggers and typed TriggerConfigs for this executor type. pub fn trigger_configs(&self) -> impl Iterator { - self.app + self.app() .triggers_with_type(Executor::TRIGGER_TYPE) .zip(&self.trigger_configs) } @@ -257,7 +258,7 @@ impl TriggerAppEngine { mut store_builder: StoreBuilder, ) -> Result<(Instance, Store)> { // Look up AppComponent - let component = self.app.get_component(component_id).with_context(|| { + let component = self.app().get_component(component_id).with_context(|| { format!( "app {:?} has no component {:?}", self.app_name, component_id From 85bab18eb63dce5d27e2412375d7ed381ae6b053 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Wed, 14 Sep 2022 13:21:57 -0400 Subject: [PATCH 21/28] Add ValuesMapBuilder::take ValuesMapBuilder::build is updated to have the more idiomatic behavior of consuming the builder, while 'take' adopts 'build's old behavor of resetting the builder to empty. Also add doc comments to all ValuesMapBuilder methods. Signed-off-by: Lann Martin --- crates/app/src/values.rs | 14 +++++++++++++- crates/trigger/src/locked.rs | 2 +- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/crates/app/src/values.rs b/crates/app/src/values.rs index 6a764a662..1e8267151 100644 --- a/crates/app/src/values.rs +++ b/crates/app/src/values.rs @@ -9,14 +9,17 @@ pub type ValuesMap = serde_json::Map; pub struct ValuesMapBuilder(ValuesMap); impl ValuesMapBuilder { + /// Returns a new empty ValuesMapBuilder. pub fn new() -> Self { Self::default() } + /// Inserts a string value into the map. pub fn string(&mut self, key: impl Into, value: impl Into) -> &mut Self { self.entry(key, value.into()) } + /// Inserts a string value into the map only if the given Option is Some. pub fn string_option( &mut self, key: impl Into, @@ -28,6 +31,7 @@ impl ValuesMapBuilder { self } + /// Inserts a string array into the map. pub fn string_array>( &mut self, key: impl Into, @@ -36,11 +40,13 @@ impl ValuesMapBuilder { self.entry(key, iter.into_iter().map(|s| s.into()).collect::>()) } + /// Inserts an entry into the map using the value's `impl Into`. pub fn entry(&mut self, key: impl Into, value: impl Into) -> &mut Self { self.0.insert(key.into(), value.into()); self } + /// Inserts an entry into the map using the value's `impl Serialize`. pub fn serializable( &mut self, key: impl Into, @@ -51,7 +57,13 @@ impl ValuesMapBuilder { Ok(self) } - pub fn build(&mut self) -> ValuesMap { + /// Returns the built ValuesMap. + pub fn build(self) -> ValuesMap { + self.0 + } + + /// Returns the build ValuesMap and resets the builder to empty. + pub fn take(&mut self) -> ValuesMap { std::mem::take(&mut self.0) } } diff --git a/crates/trigger/src/locked.rs b/crates/trigger/src/locked.rs index 0b4bebe1c..d57b709da 100644 --- a/crates/trigger/src/locked.rs +++ b/crates/trigger/src/locked.rs @@ -130,7 +130,7 @@ impl LockedAppBuilder { let metadata = ValuesMapBuilder::new() .string_option("description", component.description) .string_array("allowed_http_hosts", component.wasm.allowed_http_hosts) - .build(); + .take(); let source = { let path = match component.source { From b21f18593f6e15005e2ecf5d996c72b15d8aa529 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 15 Sep 2022 16:09:30 -0400 Subject: [PATCH 22/28] Add documentation to spin-core crate Signed-off-by: Lann Martin --- crates/core/src/host_component.rs | 57 ++++++++++++++++++++++++++++--- crates/core/src/io.rs | 2 ++ crates/core/src/lib.rs | 50 +++++++++++++++++++++++++++ crates/core/src/store.rs | 41 +++++++++++++++++++++- 4 files changed, 144 insertions(+), 6 deletions(-) diff --git a/crates/core/src/host_component.rs b/crates/core/src/host_component.rs index 6443e290a..793e65d7d 100644 --- a/crates/core/src/host_component.rs +++ b/crates/core/src/host_component.rs @@ -4,17 +4,56 @@ use anyhow::Result; use super::{Data, Linker}; +/// A trait for Spin "host components". +/// +/// A Spin host component is an interface provided to Spin components that is +/// implemented by the host. This trait is designed to be compatible with +/// [`wit-bindgen`](https://github.com/bytecodealliance/wit-bindgen)'s +/// generated bindings. +/// +/// # Example +/// +/// ```ignore +/// wit_bindgen_wasmtime::export!({paths: ["my-interface.wit"], async: *}); +/// +/// #[derive(Default)] +/// struct MyHostComponent { +/// // ... +/// } +/// +/// #[async_trait] +/// impl my_interface::MyInterface for MyHostComponent { +/// // ... +/// } +/// +/// impl HostComponent for MyHostComponent { +/// type Data = Self; +/// +/// fn add_to_linker( +/// linker: &mut Linker, +/// get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, +/// ) -> anyhow::Result<()> { +/// my_interface::add_to_linker(linker, get) +/// } +/// +/// fn build_data(&self) -> Self::Data { +/// Default::default() +/// } +/// } +/// ``` pub trait HostComponent: Send + Sync + 'static { /// Host component runtime data. type Data: Send + Sized + 'static; /// Add this component to the given Linker, using the given runtime state-getting handle. - // This function signature mirrors those generated by wit-bindgen. + /// + /// This function signature mirrors those generated by `wit-bindgen`. fn add_to_linker( linker: &mut Linker, get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, ) -> Result<()>; + /// Builds new host component runtime data for [`HostComponentsData`]. fn build_data(&self) -> Self::Data; } @@ -33,6 +72,9 @@ impl HostComponent for Arc { } } +/// An opaque handle returned by [`crate::EngineBuilder::add_host_component`] +/// which can be passed to [`HostComponentsData`] to access or set associated +/// [`HostComponent::Data`]. pub struct HostComponentDataHandle { idx: usize, _phantom: PhantomData HC::Data>, @@ -105,12 +147,21 @@ impl HostComponents { } } +/// Holds a heterogenous set of [`HostComponent::Data`]s. pub struct HostComponentsData { data: Vec>>, data_builders: Arc>, } impl HostComponentsData { + /// Sets the [`HostComponent::Data`] for the given `handle`. + pub fn set(&mut self, handle: HostComponentDataHandle, data: HC::Data) { + self.data[handle.idx] = Some(Box::new(data)); + } + + /// Retrieves a mutable reference to [`HostComponent::Data`] for the given `handle`. + /// + /// If unset, the data will be initialized with [`HostComponent::build_data`]. pub fn get_or_insert( &mut self, handle: HostComponentDataHandle, @@ -122,8 +173,4 @@ impl HostComponentsData { fn get_or_insert_idx(&mut self, idx: usize) -> &mut Box { self.data[idx].get_or_insert_with(|| self.data_builders[idx]()) } - - pub fn set(&mut self, handle: HostComponentDataHandle, data: HC::Data) { - self.data[handle.idx] = Some(Box::new(data)); - } } diff --git a/crates/core/src/io.rs b/crates/core/src/io.rs index bf0e6a667..b5f277cec 100644 --- a/crates/core/src/io.rs +++ b/crates/core/src/io.rs @@ -2,10 +2,12 @@ use std::sync::{Arc, RwLock}; use wasi_common::pipe::WritePipe; +/// An in-memory stdio output buffer. #[derive(Default)] pub struct OutputBuffer(Arc>>); impl OutputBuffer { + /// Takes the buffered output from this buffer. pub fn take(&mut self) -> Vec { std::mem::take(&mut *self.0.write().unwrap()) } diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rs index 7bc6ce021..9783f71ae 100644 --- a/crates/core/src/lib.rs +++ b/crates/core/src/lib.rs @@ -1,3 +1,12 @@ +//! Spin core execution engine +//! +//! This crate provides low-level Wasm and WASI functionality required by Spin. +//! Most of this functionality consists of wrappers around [`wasmtime`] and +//! [`wasmtime_wasi`] that narrows the flexibility of `wasmtime` to the set of +//! features used by Spin (such as only supporting `wasmtime`'s async calling style). + +#![deny(missing_docs)] + mod host_component; mod io; mod limits; @@ -14,8 +23,12 @@ pub use wasmtime::{self, Instance, Module, Trap}; use self::host_component::{HostComponents, HostComponentsBuilder}; pub use host_component::{HostComponent, HostComponentDataHandle, HostComponentsData}; +pub use io::OutputBuffer; pub use store::{Store, StoreBuilder}; +/// Global configuration for `EngineBuilder`. +/// +/// This is currently only used for advanced (undocumented) use cases. pub struct Config { inner: wasmtime::Config, } @@ -37,6 +50,7 @@ impl Default for Config { } } +/// Host state data associated with individual [Store]s and [Instance]s. pub struct Data { inner: T, wasi: WasiCtx, @@ -56,8 +70,12 @@ impl AsMut for Data { } } +/// An alias for [`wasmtime::Linker`] specialized to [`Data`]. pub type Linker = wasmtime::Linker>; +/// A builder interface for configuring a new [`Engine`]. +/// +/// A new [`EngineBuilder`] can be obtained with [`Engine::builder`]. pub struct EngineBuilder { engine: wasmtime::Engine, linker: Linker, @@ -78,6 +96,18 @@ impl EngineBuilder { }) } + /// Adds definition(s) to the built [`Engine`]. + /// + /// This method's signature is meant to be used with + /// [`wit-bindgen`](https://github.com/bytecodealliance/wit-bindgen)'s + /// generated `add_to_linker` functions, e.g.: + /// + /// ```ignore + /// wit_bindgen_wasmtime::import!({paths: ["my-interface.wit"], async: *}); + /// // ... + /// let mut builder: EngineBuilder = Engine::builder(); + /// builder.link_import(my_interface::MyInterface::add_to_linker)?; + /// ``` pub fn link_import( &mut self, f: impl FnOnce(&mut Linker, fn(&mut Data) -> &mut T) -> Result<()>, @@ -85,6 +115,11 @@ impl EngineBuilder { f(&mut self.linker, Data::as_mut) } + /// Adds a [`HostComponent`] to the built [`Engine`]. + /// + /// Returns a [`HostComponentDataHandle`] which can be passed to + /// [`HostComponentsData`] to access or set associated + /// [`HostComponent::Data`] for an instance. pub fn add_host_component( &mut self, host_component: HC, @@ -93,6 +128,11 @@ impl EngineBuilder { .add_host_component(&mut self.linker, host_component) } + /// Builds an [`Engine`] from this builder with the given host state data. + /// + /// Note that this data will generally go entirely unused, but is needed + /// by the implementation of [`Engine::instantiate_pre`]. If `T: Default`, + /// it is probably preferable to use [`EngineBuilder::build`]. pub fn build_with_data(self, instance_pre_data: T) -> Engine { let host_components = self.host_components_builder.build(); @@ -112,11 +152,14 @@ impl EngineBuilder { } impl EngineBuilder { + /// Builds an [`Engine`] from this builder. pub fn build(self) -> Engine { self.build_with_data(T::default()) } } +/// An `Engine` is a global context for the initialization and execution of +/// Spin components. pub struct Engine { inner: wasmtime::Engine, linker: Linker, @@ -125,14 +168,17 @@ pub struct Engine { } impl Engine { + /// Creates a new [`EngineBuilder`] with the given [`Config`]. pub fn builder(config: &Config) -> Result> { EngineBuilder::new(config) } + /// Creates a new [`StoreBuilder`]. pub fn store_builder(&self) -> StoreBuilder { StoreBuilder::new(self.inner.clone(), &self.host_components) } + /// Creates a new [`InstancePre`] for the given [`Module`]. #[instrument(skip_all)] pub fn instantiate_pre(&self, module: &Module) -> Result> { let mut store = self.instance_pre_store.lock().unwrap(); @@ -147,11 +193,15 @@ impl AsRef for Engine { } } +/// A pre-initialized instance that is ready to be instantiated. +/// +/// See [`wasmtime::InstancePre`] for more information. pub struct InstancePre { inner: wasmtime::InstancePre>, } impl InstancePre { + /// Instantiates this instance with the given [`Store`]. #[instrument(skip_all)] pub async fn instantiate_async(&self, store: &mut Store) -> Result { self.inner.instantiate_async(store).await diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs index 9d87a39df..e86ff3f58 100644 --- a/crates/core/src/store.rs +++ b/crates/core/src/store.rs @@ -16,11 +16,18 @@ use super::{ Data, }; +/// A `Store` holds the runtime state of a Spin instance. +/// +/// In general, a `Store` is expected to live only for the lifetime of a single +/// Spin trigger invocation. +/// +/// A `Store` can be built with a [`StoreBuilder`]. pub struct Store { inner: wasmtime::Store>, } impl Store { + /// Returns a mutable reference to the [`HostComponentsData`] of this [`Store`]. pub fn host_components_data(&mut self) -> &mut HostComponentsData { &mut self.inner.data_mut().host_components_data } @@ -70,6 +77,9 @@ const READ_ONLY_FILE_CAPS: FileCaps = FileCaps::from_bits_truncate( | FileCaps::POLL_READWRITE.bits(), ); +/// A builder interface for configuring a new [`Store`]. +/// +/// A new [`StoreBuilder`] can be obtained with [`crate::Engine::store_builder`]. pub struct StoreBuilder { engine: wasmtime::Engine, wasi: std::result::Result, String>, @@ -79,6 +89,7 @@ pub struct StoreBuilder { } impl StoreBuilder { + // Called by Engine::store_builder. pub(crate) fn new(engine: wasmtime::Engine, host_components: &HostComponents) -> Self { Self { engine, @@ -89,50 +100,65 @@ impl StoreBuilder { } } + /// Sets a maximum memory allocation limit. + /// + /// See [`wasmtime::ResourceLimiter::memory_growing`] (`maximum`) for + /// details on how this limit is enforced. pub fn max_memory_size(&mut self, max_memory_size: usize) { self.store_limits = StoreLimitsAsync::new(Some(max_memory_size), None); } + /// Inherit stdin, stdout, and stderr from the host process. pub fn inherit_stdio(&mut self) { self.with_wasi(|wasi| wasi.inherit_stdio()); } + /// Sets the WASI `stdin` descriptor. pub fn stdin(&mut self, file: impl WasiFile + 'static) { self.with_wasi(|wasi| wasi.stdin(Box::new(file))) } + /// Sets the WASI `stdin` descriptor to the given [`Read`]er. pub fn stdin_pipe(&mut self, r: impl Read + Send + Sync + 'static) { self.stdin(ReadPipe::new(r)) } + /// Sets the WASI `stdout` descriptor. pub fn stdout(&mut self, file: impl WasiFile + 'static) { self.with_wasi(|wasi| wasi.stdout(Box::new(file))) } + /// Sets the WASI `stdout` descriptor to the given [`Write`]er. pub fn stdout_pipe(&mut self, w: impl Write + Send + Sync + 'static) { self.stdout(WritePipe::new(w)) } - + /// Sets the WASI `stdout` descriptor to an in-memory buffer which can be + /// retrieved after execution from the returned [`OutputBuffer`]. pub fn stdout_buffered(&mut self) -> OutputBuffer { let buffer = OutputBuffer::default(); self.stdout(buffer.writer()); buffer } + /// Sets the WASI `stderr` descriptor. pub fn stderr(&mut self, file: impl WasiFile + 'static) { self.with_wasi(|wasi| wasi.stderr(Box::new(file))) } + /// Sets the WASI `stderr` descriptor to the given [`Write`]er. pub fn stderr_pipe(&mut self, w: impl Write + Send + Sync + 'static) { self.stderr(WritePipe::new(w)) } + /// Sets the WASI `stderr` descriptor to an in-memory buffer which can be + /// retrieved after execution from the returned [`OutputBuffer`]. pub fn stderr_buffered(&mut self) -> OutputBuffer { let buffer = OutputBuffer::default(); self.stderr(buffer.writer()); buffer } + /// Appends the given strings to the the WASI 'args'. pub fn args<'b>(&mut self, args: impl IntoIterator) -> Result<()> { self.try_with_wasi(|mut wasi| { for arg in args { @@ -142,6 +168,7 @@ impl StoreBuilder { }) } + /// Sets the given key/value string entries on the the WASI 'env'. pub fn env( &mut self, vars: impl IntoIterator, impl AsRef)>, @@ -154,6 +181,8 @@ impl StoreBuilder { }) } + /// "Mounts" the given `host_path` into the WASI filesystem at the given + /// `guest_path` with read-only capabilities. pub fn read_only_preopened_dir( &mut self, host_path: impl AsRef, @@ -166,6 +195,8 @@ impl StoreBuilder { Ok(()) } + /// "Mounts" the given `host_path` into the WASI filesystem at the given + /// `guest_path` with read and write capabilities. pub fn read_write_preopened_dir( &mut self, host_path: impl AsRef, @@ -175,10 +206,14 @@ impl StoreBuilder { self.try_with_wasi(|wasi| wasi.preopened_dir(dir, guest_path)) } + /// Returns a mutable reference to the built pub fn host_components_data(&mut self) -> &mut HostComponentsData { &mut self.host_components_data } + /// Builds a [`Store`] from this builder with given host state data. + /// + /// If `T: Default`, it may be preferable to use [`StoreBuilder::build`]. pub fn build_with_data(self, inner_data: T) -> Result> { let mut wasi = self.wasi.map_err(anyhow::Error::msg)?.unwrap().build(); @@ -202,10 +237,14 @@ impl StoreBuilder { Ok(Store { inner }) } + /// Builds a [`Store`] from this builder with `Default` host state data. pub fn build(self) -> Result> { self.build_with_data(T::default()) } + // Helpers for adapting the "consuming builder" style of WasiCtxBuilder to + // StoreBuilder's "non-consuming builder" style. + fn with_wasi(&mut self, f: impl FnOnce(WasiCtxBuilder) -> WasiCtxBuilder) { let _ = self.try_with_wasi(|wasi| Ok(f(wasi))); } From ddd85b72f77a88f8961f45c1bbac312b183fda9b Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Thu, 15 Sep 2022 17:08:42 -0400 Subject: [PATCH 23/28] Add docs to spin-app crate Signed-off-by: Lann Martin --- crates/app/src/host_component.rs | 8 +++ crates/app/src/lib.rs | 94 +++++++++++++++++++++++++++++--- crates/app/src/locked.rs | 6 +- crates/app/src/values.rs | 4 +- 4 files changed, 103 insertions(+), 9 deletions(-) diff --git a/crates/app/src/host_component.rs b/crates/app/src/host_component.rs index c428aefe4..cc81e6698 100644 --- a/crates/app/src/host_component.rs +++ b/crates/app/src/host_component.rs @@ -4,7 +4,15 @@ use spin_core::{EngineBuilder, HostComponent, HostComponentsData}; use crate::AppComponent; +/// A trait for "dynamic" Spin host components. +/// +/// This extends [`HostComponent`] to support per-[`AppComponent`] dynamic +/// runtime configuration. pub trait DynamicHostComponent: HostComponent { + /// Called on [`AppComponent`] instance initialization. + /// + /// The `data` returned by [`HostComponent::build_data`] is passed, along + /// with a reference to the `component` being instantiated. fn update_data(&self, data: &mut Self::Data, component: &AppComponent) -> anyhow::Result<()>; } diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index 353a6c0bd..a5a3a9a74 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -1,3 +1,11 @@ +//! Spin internal application interfaces +//! +//! This crate contains interfaces to Spin application configuration to be used +//! by crates that implement Spin execution environments: trigger executors and +//! host components, in particular. + +#![deny(missing_docs)] + mod host_component; pub mod locked; pub mod values; @@ -13,17 +21,25 @@ pub use async_trait::async_trait; pub use host_component::DynamicHostComponent; pub use locked::Variable; +/// A trait for implementing the low-level operations needed to load an [`App`]. // TODO(lann): Should this migrate to spin-loader? #[async_trait] pub trait Loader { + /// Called with an implementation-defined `uri` pointing to some + /// representation of a [`LockedApp`], which will be loaded. async fn load_app(&self, uri: &str) -> anyhow::Result; + /// Called with a [`LockedComponentSource`] pointing to a Wasm module + /// binary, which will be loaded. async fn load_module( &self, engine: &wasmtime::Engine, source: &LockedComponentSource, ) -> anyhow::Result; + /// Called with an [`AppComponent`]; any `files` configured with the + /// component should be "mounted" into the `store_builder`, via e.g. + /// [`StoreBuilder::read_only_preopened_dir`]. async fn mount_files( &self, store_builder: &mut StoreBuilder, @@ -31,12 +47,15 @@ pub trait Loader { ) -> anyhow::Result<()>; } +/// An `AppLoader` holds an implementation of [`Loader`] along with +/// [`DynamicHostComponents`] configuration. pub struct AppLoader { inner: Box, dynamic_host_components: DynamicHostComponents, } impl AppLoader { + /// Creates a new [`AppLoader`]. pub fn new(loader: impl Loader + Send + Sync + 'static) -> Self { Self { inner: Box::new(loader), @@ -44,6 +63,11 @@ impl AppLoader { } } + /// Adds a [`DynamicHostComponent`] to the given [`EngineBuilder`] and + /// configures this [`AppLoader`] to update it on component instantiation. + /// + /// This calls [`EngineBuilder::add_host_component`] for you; it should not + /// be called separately. pub fn add_dynamic_host_component( &mut self, engine_builder: &mut EngineBuilder, @@ -53,6 +77,7 @@ impl AppLoader { .add_dynamic_host_component(engine_builder, host_component) } + /// Loads an [`App`] from the given `Loader`-implementation-specific `uri`. pub async fn load_app(&self, uri: String) -> Result { let locked = self .inner @@ -66,6 +91,8 @@ impl AppLoader { }) } + /// Loads an [`OwnedApp`] from the given `Loader`-implementation-specific + /// `uri`; the [`OwnedApp`] takes ownership of this [`AppLoader`]. pub async fn load_owned_app(self, uri: String) -> Result { OwnedApp::try_new_async(self, |loader| Box::pin(loader.load_app(uri))).await } @@ -88,11 +115,13 @@ pub struct OwnedApp { } impl OwnedApp { + /// Returns a reference to the owned [`App`]. pub fn borrowed(&self) -> &App { self.borrow_app() } } +/// An `App` holds loaded configuration for a Spin application. #[derive(Debug)] pub struct App<'a> { loader: &'a AppLoader, @@ -101,10 +130,16 @@ pub struct App<'a> { } impl<'a> App<'a> { + /// Returns a [`Loader`]-implementation-specific URI for this app. pub fn uri(&self) -> &str { &self.uri } + /// Deserializes typed metadata for this app. + /// + /// Returns `Ok(None)` if there is no metadata for the given `key` and an + /// `Err` only if there _is_ a value for the `key` but the typed + /// deserialization failed. pub fn get_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result> { self.locked .metadata @@ -113,15 +148,21 @@ impl<'a> App<'a> { .transpose() } + /// Deserializes typed metadata for this app. + /// + /// Like [`App::get_metadata`], but returns an `Err` if there is no metadata + /// for the given `key`. pub fn require_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result { self.get_metadata(key)? - .ok_or_else(|| Error::ManifestError(format!("missing required {key:?}"))) + .ok_or_else(|| Error::MetadataError(format!("missing required {key:?}"))) } + /// Returns an iterator of custom config [`Variable`]s defined for this app. pub fn variables(&self) -> impl Iterator { self.locked.variables.iter() } + /// Returns an iterator of [`AppComponent`]s defined for this app. pub fn components(&self) -> impl Iterator { self.locked .components @@ -129,11 +170,14 @@ impl<'a> App<'a> { .map(|locked| AppComponent { app: self, locked }) } + /// Returns the [`AppComponent`] with the given `component_id`, or `None` + /// if it doesn't exist. pub fn get_component(&self, component_id: &str) -> Option { self.components() .find(|component| component.locked.id == component_id) } + /// Returns an iterator of [`AppTrigger`]s defined for this app. pub fn triggers(&self) -> impl Iterator { self.locked .triggers @@ -141,37 +185,50 @@ impl<'a> App<'a> { .map(|locked| AppTrigger { app: self, locked }) } + /// Returns an iterator of [`AppTrigger`]s defined for this app with + /// the given `trigger_type`. pub fn triggers_with_type(&'a self, trigger_type: &'a str) -> impl Iterator { self.triggers() .filter(move |trigger| trigger.locked.trigger_type == trigger_type) } } +/// An `AppComponent` holds configuration for a Spin application component. pub struct AppComponent<'a> { + /// The app this component belongs to. pub app: &'a App<'a>, locked: &'a LockedComponent, } impl<'a> AppComponent<'a> { + /// Returns this component's app-unique ID. pub fn id(&self) -> &str { &self.locked.id } + /// Returns this component's Wasm module source. pub fn source(&self) -> &LockedComponentSource { &self.locked.source } + /// Returns an iterator of [`ContentPath`]s for this component's configured + /// "directory mounts". pub fn files(&self) -> std::slice::Iter { self.locked.files.iter() } + /// Deserializes typed metadata for this component. + /// + /// Returns `Ok(None)` if there is no metadata for the given `key` and an + /// `Err` only if there _is_ a value for the `key` but the typed + /// deserialization failed. pub fn get_metadata>(&self, key: &str) -> Result> { self.locked .metadata .get(key) .map(|value| { T::deserialize(value).map_err(|err| { - Error::ManifestError(format!( + Error::MetadataError(format!( "failed to deserialize {key:?} = {value:?}: {err:?}" )) }) @@ -179,10 +236,12 @@ impl<'a> AppComponent<'a> { .transpose() } + /// Returns an iterator of custom config values for this component. pub fn config(&self) -> impl Iterator { self.locked.config.iter() } + /// Loads and returns the [`spin_core::Module`] for this component. pub async fn load_module( &self, engine: &Engine, @@ -195,6 +254,11 @@ impl<'a> AppComponent<'a> { .map_err(Error::LoaderError) } + /// Updates the given [`StoreBuilder`] with configuration for this component. + /// + /// In particular, the WASI 'env' and "preloaded dirs" are set up, and any + /// [`DynamicHostComponent`]s associated with the source [`AppLoader`] are + /// configured. pub async fn apply_store_config(&self, builder: &mut StoreBuilder) -> Result<()> { builder.env(&self.locked.env).map_err(Error::CoreError)?; @@ -214,58 +278,74 @@ impl<'a> AppComponent<'a> { } } +/// An `AppTrigger` holds configuration for a Spin application trigger. pub struct AppTrigger<'a> { + /// The app this trigger belongs to. pub app: &'a App<'a>, locked: &'a LockedTrigger, } impl<'a> AppTrigger<'a> { + /// Returns this trigger's app-unique ID. pub fn id(&self) -> &str { &self.locked.id } + /// Returns the Trigger's type. pub fn trigger_type(&self) -> &str { &self.locked.trigger_type } + /// Returns a reference to the [`AppComponent`] configured for this trigger. + /// + /// This is a convenience wrapper that looks up the component based on the + /// 'component' metadata value which is conventionally a component ID. pub fn component(&self) -> Result> { let component_id = self.locked.trigger_config.get("component").ok_or_else(|| { - Error::ManifestError(format!( + Error::MetadataError(format!( "trigger {:?} missing 'component' config field", self.locked.id )) })?; let component_id = component_id.as_str().ok_or_else(|| { - Error::ManifestError(format!( + Error::MetadataError(format!( "trigger {:?} 'component' field has unexpected value {:?}", self.locked.id, component_id )) })?; self.app.get_component(component_id).ok_or_else(|| { - Error::ManifestError(format!( + Error::MetadataError(format!( "missing component {:?} configured for trigger {:?}", component_id, self.locked.id )) }) } + /// Deserializes this trigger's configuration into a typed value. pub fn typed_config>(&self) -> Result { Ok(Config::deserialize(&self.locked.trigger_config)?) } } +/// Type alias for a [`Result`]s with [`Error`]. pub type Result = std::result::Result; +/// Errors returned by methods in this crate. #[derive(Debug, thiserror::Error)] pub enum Error { + /// An error propagated from the [`spin_core`] crate. #[error("spin core error: {0:#}")] CoreError(anyhow::Error), + /// An error from a [`DynamicHostComponent`]. #[error("host component error: {0:#}")] HostComponentError(anyhow::Error), + /// An error from a [`Loader`] implementation. #[error("loader error: {0:#}")] LoaderError(anyhow::Error), - #[error("manifest error: {0}")] - ManifestError(String), + /// An error indicating missing or unexpected metadata. + #[error("metadata error: {0}")] + MetadataError(String), + /// An error indicating failed JSON (de)serialization. #[error("json error: {0}")] JsonError(#[from] serde_json::Error), } diff --git a/crates/app/src/locked.rs b/crates/app/src/locked.rs index 83c90bebb..fd5a8e3de 100644 --- a/crates/app/src/locked.rs +++ b/crates/app/src/locked.rs @@ -1,3 +1,5 @@ +//! Spin lock file (spin.lock) serialization models. + use std::path::PathBuf; use serde::{Deserialize, Serialize}; @@ -5,7 +7,7 @@ use serde_json::Value; use crate::values::ValuesMap; -// LockedMap gives deterministic encoding, which we want. +/// A String-keyed map with deterministic serialization order. pub type LockedMap = std::collections::BTreeMap; /// A LockedApp represents a "fully resolved" Spin application. @@ -26,10 +28,12 @@ pub struct LockedApp { } impl LockedApp { + /// Deserializes a [`LockedApp`] from the given JSON data. pub fn from_json(contents: &[u8]) -> serde_json::Result { serde_json::from_slice(contents) } + /// Serializes the [`LockedApp`] into JSON data. pub fn to_json(&self) -> serde_json::Result> { serde_json::to_vec_pretty(&self) } diff --git a/crates/app/src/values.rs b/crates/app/src/values.rs index 1e8267151..b2a121a5b 100644 --- a/crates/app/src/values.rs +++ b/crates/app/src/values.rs @@ -1,7 +1,9 @@ +//! Dynamically-typed value helpers. + use serde::Serialize; use serde_json::Value; -// ValuesMap stores dynamically-typed values. +/// A String-keyed map with dynamically-typed values. pub type ValuesMap = serde_json::Map; /// ValuesMapBuilder assists in building a ValuesMap. From c1f831d8e3e4b80d0dc3b283512287142dea9730 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 16 Sep 2022 09:54:08 -0400 Subject: [PATCH 24/28] Remove '--include-ignored' from Makefile It isn't clear why this was included in the first place and it doesn't appear to be used now. Signed-off-by: Lann Martin --- Makefile | 6 +++--- crates/redis/src/tests.rs | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index a8d02d84e..32010a11c 100644 --- a/Makefile +++ b/Makefile @@ -22,16 +22,16 @@ check-rust-examples: .PHONY: test-unit test-unit: - RUST_LOG=$(LOG_LEVEL) cargo test --all --no-fail-fast -- --skip integration_tests --nocapture --include-ignored + RUST_LOG=$(LOG_LEVEL) cargo test --all --no-fail-fast -- --skip integration_tests --nocapture .PHONY: test-integration test-integration: - RUST_LOG=$(LOG_LEVEL) cargo test --test integration --no-fail-fast -- --nocapture --include-ignored + RUST_LOG=$(LOG_LEVEL) cargo test --test integration --no-fail-fast -- --nocapture .PHONY: test-e2e test-e2e: RUST_LOG=$(LOG_LEVEL) cargo test --test integration --features e2e-tests --no-fail-fast -- integration_tests::test_dependencies --nocapture - RUST_LOG=$(LOG_LEVEL) cargo test --test integration --features e2e-tests --no-fail-fast -- --skip integration_tests::test_dependencies --nocapture --include-ignored + RUST_LOG=$(LOG_LEVEL) cargo test --test integration --features e2e-tests --no-fail-fast -- --skip integration_tests::test_dependencies --nocapture .PHONY: test-sdk-go test-sdk-go: diff --git a/crates/redis/src/tests.rs b/crates/redis/src/tests.rs index fdd75f0fd..951ae655d 100644 --- a/crates/redis/src/tests.rs +++ b/crates/redis/src/tests.rs @@ -12,7 +12,6 @@ fn create_trigger_event(channel: &str, payload: &str) -> redis::Msg { .unwrap() } -#[ignore] #[tokio::test] async fn test_pubsub() -> Result<()> { let trigger: RedisTrigger = TestConfig::default() From 6934177ec659cc67b7df25f6c1d3d526f1828c8d Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 16 Sep 2022 14:00:07 -0400 Subject: [PATCH 25/28] Deduplicate App::get_metadata, AppComponent::get_metadata Also add AppComponent::require_metadata for interface consistency. Signed-off-by: Lann Martin --- Cargo.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index bc2b5d589..860023a09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3172,7 +3172,7 @@ name = "spin-abi-conformance" version = "0.5.0" dependencies = [ "anyhow", - "cap-std 0.25.2", + "cap-std", "clap 3.2.19", "rand 0.8.5", "rand_chacha 0.3.1", From 47a89c7b2e34776d09a254626b3eba5a2ce22e27 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Fri, 16 Sep 2022 14:00:07 -0400 Subject: [PATCH 26/28] Deduplicate App::get_metadata, AppComponent::get_metadata Also add AppComponent::require_metadata for interface consistency. Signed-off-by: Lann Martin --- crates/app/src/lib.rs | 34 ++++++++++++++-------------------- crates/app/src/values.rs | 31 ++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/crates/app/src/lib.rs b/crates/app/src/lib.rs index a5a3a9a74..e01fb95d3 100644 --- a/crates/app/src/lib.rs +++ b/crates/app/src/lib.rs @@ -16,6 +16,7 @@ use spin_core::{wasmtime, Engine, EngineBuilder, StoreBuilder}; use host_component::DynamicHostComponents; use locked::{ContentPath, LockedApp, LockedComponent, LockedComponentSource, LockedTrigger}; +use values::MetadataExt; pub use async_trait::async_trait; pub use host_component::DynamicHostComponent; @@ -141,20 +142,15 @@ impl<'a> App<'a> { /// `Err` only if there _is_ a value for the `key` but the typed /// deserialization failed. pub fn get_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result> { - self.locked - .metadata - .get(key) - .map(|value| Ok(T::deserialize(value)?)) - .transpose() + self.locked.metadata.get_typed(key) } /// Deserializes typed metadata for this app. /// - /// Like [`App::get_metadata`], but returns an `Err` if there is no metadata - /// for the given `key`. + /// Like [`App::get_metadata`], but returns an error if there is + /// no metadata for the given `key`. pub fn require_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result { - self.get_metadata(key)? - .ok_or_else(|| Error::MetadataError(format!("missing required {key:?}"))) + self.locked.metadata.require_typed(key) } /// Returns an iterator of custom config [`Variable`]s defined for this app. @@ -223,17 +219,15 @@ impl<'a> AppComponent<'a> { /// `Err` only if there _is_ a value for the `key` but the typed /// deserialization failed. pub fn get_metadata>(&self, key: &str) -> Result> { - self.locked - .metadata - .get(key) - .map(|value| { - T::deserialize(value).map_err(|err| { - Error::MetadataError(format!( - "failed to deserialize {key:?} = {value:?}: {err:?}" - )) - }) - }) - .transpose() + self.locked.metadata.get_typed(key) + } + + /// Deserializes typed metadata for this component. + /// + /// Like [`AppComponent::get_metadata`], but returns an error if there is + /// no metadata for the given `key`. + pub fn require_metadata<'this, T: Deserialize<'this>>(&'this self, key: &str) -> Result { + self.locked.metadata.require_typed(key) } /// Returns an iterator of custom config values for this component. diff --git a/crates/app/src/values.rs b/crates/app/src/values.rs index b2a121a5b..a1434a5ae 100644 --- a/crates/app/src/values.rs +++ b/crates/app/src/values.rs @@ -1,8 +1,10 @@ //! Dynamically-typed value helpers. -use serde::Serialize; +use serde::{Deserialize, Serialize}; use serde_json::Value; +use crate::Error; + /// A String-keyed map with dynamically-typed values. pub type ValuesMap = serde_json::Map; @@ -69,3 +71,30 @@ impl ValuesMapBuilder { std::mem::take(&mut self.0) } } + +pub(crate) trait MetadataExt { + fn get_value(&self, key: impl AsRef) -> Option<&Value>; + + fn get_typed<'a, T: Deserialize<'a>>( + &'a self, + key: impl AsRef, + ) -> Result, Error> { + let key = key.as_ref(); + self.get_value(key) + .map(|value| T::deserialize(value)) + .transpose() + .map_err(|err| Error::MetadataError(format!("invalid value for {key:?}: {err:?}"))) + } + + fn require_typed<'a, T: Deserialize<'a>>(&'a self, key: impl AsRef) -> Result { + let key = key.as_ref(); + self.get_typed(key)? + .ok_or_else(|| Error::MetadataError(format!("missing required {key:?}"))) + } +} + +impl MetadataExt for ValuesMap { + fn get_value(&self, key: impl AsRef) -> Option<&Value> { + self.get(key.as_ref()) + } +} From 4ee7dc74a6cb7272de2a483d333e5c28008c092f Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Mon, 19 Sep 2022 09:32:40 -0400 Subject: [PATCH 27/28] Add tests to spin-core crate Signed-off-by: Lann Martin --- Cargo.lock | 2 + build.rs | 1 + crates/core/Cargo.toml | 6 +- crates/core/src/host_component.rs | 41 ++++ crates/core/src/io.rs | 19 ++ crates/core/src/store.rs | 16 +- .../tests/core-wasi-test/.cargo/config.toml | 2 + crates/core/tests/core-wasi-test/Cargo.toml | 9 + crates/core/tests/core-wasi-test/src/main.rs | 58 +++++ crates/core/tests/integration_test.rs | 210 ++++++++++++++++++ 10 files changed, 360 insertions(+), 4 deletions(-) create mode 100644 crates/core/tests/core-wasi-test/.cargo/config.toml create mode 100644 crates/core/tests/core-wasi-test/Cargo.toml create mode 100644 crates/core/tests/core-wasi-test/src/main.rs create mode 100644 crates/core/tests/integration_test.rs diff --git a/Cargo.lock b/Cargo.lock index 860023a09..ec44c4eb1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3288,6 +3288,8 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", + "tempfile", + "tokio", "tracing", "wasi-cap-std-sync", "wasi-common", diff --git a/build.rs b/build.rs index 10bff1f6b..58f52f5f3 100644 --- a/build.rs +++ b/build.rs @@ -39,6 +39,7 @@ error: the `wasm32-wasi` target is not installed std::fs::create_dir_all("target/test-programs").unwrap(); + build_wasm_test_program("core-wasi-test.wasm", "crates/core/tests/core-wasi-test"); build_wasm_test_program("rust-http-test.wasm", "crates/http/tests/rust-http-test"); build_wasm_test_program("redis-rust.wasm", "crates/redis/tests/rust"); build_wasm_test_program("wagi-test.wasm", "crates/http/tests/wagi-test"); diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.toml index d9308e39f..f9824c99c 100644 --- a/crates/core/Cargo.toml +++ b/crates/core/Cargo.toml @@ -10,4 +10,8 @@ async-trait = "0.1" wasi-cap-std-sync = "0.39" wasi-common = "0.39" wasmtime = "0.39" -wasmtime-wasi = { version = "0.39", features = ["tokio"] } \ No newline at end of file +wasmtime-wasi = { version = "0.39", features = ["tokio"] } + +[dev-dependencies] +tempfile = "3" +tokio = { version = "1", features = ["macros", "rt"] } \ No newline at end of file diff --git a/crates/core/src/host_component.rs b/crates/core/src/host_component.rs index 793e65d7d..ea940e4a4 100644 --- a/crates/core/src/host_component.rs +++ b/crates/core/src/host_component.rs @@ -174,3 +174,44 @@ impl HostComponentsData { self.data[idx].get_or_insert_with(|| self.data_builders[idx]()) } } + +#[cfg(test)] +mod tests { + use super::*; + + struct TestHC; + + impl HostComponent for TestHC { + type Data = u8; + + fn add_to_linker( + _linker: &mut Linker, + _get: impl Fn(&mut Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> Result<()> { + Ok(()) + } + + fn build_data(&self) -> Self::Data { + 0 + } + } + + #[test] + fn host_components_data() { + let engine = wasmtime::Engine::default(); + let mut linker: crate::Linker<()> = crate::Linker::new(&engine); + + let mut builder = HostComponents::builder(); + let handle1 = builder + .add_host_component(&mut linker, Arc::new(TestHC)) + .unwrap(); + let handle2 = builder.add_host_component(&mut linker, TestHC).unwrap(); + let host_components = builder.build(); + let mut hc_data = host_components.new_data(); + + assert_eq!(hc_data.get_or_insert(handle1), &0); + + hc_data.set(handle2, 1); + assert_eq!(hc_data.get_or_insert(handle2), &1); + } +} diff --git a/crates/core/src/io.rs b/crates/core/src/io.rs index b5f277cec..b57027d29 100644 --- a/crates/core/src/io.rs +++ b/crates/core/src/io.rs @@ -16,3 +16,22 @@ impl OutputBuffer { WritePipe::from_shared(self.0.clone()) } } + +#[cfg(test)] +mod tests { + use std::io::IoSlice; + + use wasi_common::WasiFile; + + use super::*; + + #[tokio::test] + async fn take_what_you_write() { + let mut buf = OutputBuffer::default(); + buf.writer() + .write_vectored(&[IoSlice::new(b"foo")]) + .await + .unwrap(); + assert_eq!(buf.take(), b"foo"); + } +} diff --git a/crates/core/src/store.rs b/crates/core/src/store.rs index e86ff3f58..698eb85d6 100644 --- a/crates/core/src/store.rs +++ b/crates/core/src/store.rs @@ -108,9 +108,9 @@ impl StoreBuilder { self.store_limits = StoreLimitsAsync::new(Some(max_memory_size), None); } - /// Inherit stdin, stdout, and stderr from the host process. - pub fn inherit_stdio(&mut self) { - self.with_wasi(|wasi| wasi.inherit_stdio()); + /// Inherit stdin from the host process. + pub fn inherit_stdin(&mut self) { + self.with_wasi(|wasi| wasi.inherit_stdin()); } /// Sets the WASI `stdin` descriptor. @@ -123,6 +123,11 @@ impl StoreBuilder { self.stdin(ReadPipe::new(r)) } + /// Inherit stdin from the host process. + pub fn inherit_stdout(&mut self) { + self.with_wasi(|wasi| wasi.inherit_stdout()); + } + /// Sets the WASI `stdout` descriptor. pub fn stdout(&mut self, file: impl WasiFile + 'static) { self.with_wasi(|wasi| wasi.stdout(Box::new(file))) @@ -140,6 +145,11 @@ impl StoreBuilder { buffer } + /// Inherit stdin from the host process. + pub fn inherit_stderr(&mut self) { + self.with_wasi(|wasi| wasi.inherit_stderr()); + } + /// Sets the WASI `stderr` descriptor. pub fn stderr(&mut self, file: impl WasiFile + 'static) { self.with_wasi(|wasi| wasi.stderr(Box::new(file))) diff --git a/crates/core/tests/core-wasi-test/.cargo/config.toml b/crates/core/tests/core-wasi-test/.cargo/config.toml new file mode 100644 index 000000000..6b77899cb --- /dev/null +++ b/crates/core/tests/core-wasi-test/.cargo/config.toml @@ -0,0 +1,2 @@ +[build] +target = "wasm32-wasi" diff --git a/crates/core/tests/core-wasi-test/Cargo.toml b/crates/core/tests/core-wasi-test/Cargo.toml new file mode 100644 index 000000000..38d3d866c --- /dev/null +++ b/crates/core/tests/core-wasi-test/Cargo.toml @@ -0,0 +1,9 @@ +[package] +name = "core-wasi-test" +version = "0.1.0" +edition = "2021" + +[profile.release] +debug = true + +[workspace] \ No newline at end of file diff --git a/crates/core/tests/core-wasi-test/src/main.rs b/crates/core/tests/core-wasi-test/src/main.rs new file mode 100644 index 000000000..97dcc11ac --- /dev/null +++ b/crates/core/tests/core-wasi-test/src/main.rs @@ -0,0 +1,58 @@ +//! This test program takes argument(s) that determine which WASI feature to +//! exercise and returns an exit code of 0 for success, 1 for WASI interface +//! failure (which is sometimes expected in a test), and some other code on +//! invalid argument(s). + +#[link(wasm_import_module = "multiplier")] +extern "C" { + fn multiply(n: i32) -> i32; +} + +type Result = std::result::Result<(), Box>; + +fn main() -> Result { + let mut args = std::env::args(); + let cmd = args.next().expect("cmd"); + match cmd.as_str() { + "noop" => (), + "echo" => { + eprintln!("echo"); + std::io::copy(&mut std::io::stdin(), &mut std::io::stdout())?; + } + "alloc" => { + let size: usize = args.next().expect("size").parse().expect("size"); + eprintln!("alloc {size}"); + let layout = std::alloc::Layout::from_size_align(size, 8).expect("layout"); + unsafe { + let p = std::alloc::alloc(layout); + if p.is_null() { + return Err("allocation failed".into()); + } + // Force allocation to actually happen + p.read_volatile(); + } + } + "read" => { + let path = args.next().expect("path"); + eprintln!("read {path}"); + std::fs::read(path)?; + } + "write" => { + let path = args.next().expect("path"); + eprintln!("write {path}"); + std::fs::write(path, "content")?; + } + "multiply" => { + let input: i32 = args.next().expect("input").parse().expect("i32"); + eprintln!("multiply {input}"); + let output = unsafe { multiply(input) }; + println!("{output}"); + } + "panic" => { + eprintln!("panic"); + panic!("intentional panic"); + } + cmd => panic!("unknown cmd {cmd}"), + }; + Ok(()) +} diff --git a/crates/core/tests/integration_test.rs b/crates/core/tests/integration_test.rs new file mode 100644 index 000000000..94be7e0ed --- /dev/null +++ b/crates/core/tests/integration_test.rs @@ -0,0 +1,210 @@ +use std::{io::Cursor, path::PathBuf}; + +use spin_core::{Config, Engine, HostComponent, Module, StoreBuilder, Trap}; +use tempfile::TempDir; +use wasmtime::TrapCode; + +#[tokio::test(flavor = "multi_thread")] +async fn test_stdio() { + let stdout = run_core_wasi_test(["echo"], |store_builder| { + store_builder.stdin_pipe(Cursor::new(b"DATA")); + }) + .await + .unwrap(); + + assert_eq!(stdout, "DATA"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_read_only_preopened_dir() { + let filename = "test_file"; + let tempdir = TempDir::new().unwrap(); + std::fs::write(tempdir.path().join(filename), "x").unwrap(); + + run_core_wasi_test(["read", filename], |store_builder| { + store_builder + .read_only_preopened_dir(&tempdir, "/".into()) + .unwrap(); + }) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_read_only_preopened_dir_write_fails() { + let filename = "test_file"; + let tempdir = TempDir::new().unwrap(); + std::fs::write(tempdir.path().join(filename), "x").unwrap(); + + let err = run_core_wasi_test(["write", filename], |store_builder| { + store_builder + .read_only_preopened_dir(&tempdir, "/".into()) + .unwrap(); + }) + .await + .unwrap_err(); + let trap = err.downcast::().expect("trap"); + assert_eq!(trap.i32_exit_status(), Some(1)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_read_write_preopened_dir() { + let filename = "test_file"; + let tempdir = TempDir::new().unwrap(); + + run_core_wasi_test(["write", filename], |store_builder| { + store_builder + .read_write_preopened_dir(&tempdir, "/".into()) + .unwrap(); + }) + .await + .unwrap(); + + let content = std::fs::read(tempdir.path().join(filename)).unwrap(); + assert_eq!(content, b"content"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_max_memory_size_obeyed() { + let max = 10_000_000; + let alloc = max / 10; + run_core_wasi_test(["alloc", &format!("{alloc}")], |store_builder| { + store_builder.max_memory_size(max); + }) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_max_memory_size_violated() { + let max = 10_000_000; + let alloc = max * 2; + let err = run_core_wasi_test(["alloc", &format!("{alloc}")], |store_builder| { + store_builder.max_memory_size(max); + }) + .await + .unwrap_err(); + let trap = err.downcast::().expect("trap"); + assert_eq!(trap.i32_exit_status(), Some(1)); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_host_component() { + let stdout = run_core_wasi_test(["multiply", "5"], |_| {}).await.unwrap(); + assert_eq!(stdout, "10"); +} + +#[tokio::test(flavor = "multi_thread")] +async fn test_host_component_data_update() { + // Need to build Engine separately to get the HostComponentDataHandle + let mut engine_builder = Engine::builder(&test_config()).unwrap(); + let factor_data_handle = engine_builder + .add_host_component(MultiplierHostComponent) + .unwrap(); + let engine: Engine<()> = engine_builder.build(); + + let stdout = run_core_wasi_test_engine(&engine, ["multiply", "5"], |store_builder| { + store_builder + .host_components_data() + .set(factor_data_handle, 100); + }) + .await + .unwrap(); + assert_eq!(stdout, "500"); +} + +#[tokio::test(flavor = "multi_thread")] +#[cfg(not(tarpaulin))] +async fn test_panic() { + let err = run_core_wasi_test(["panic"], |_| {}).await.unwrap_err(); + let trap = err.downcast::().expect("trap"); + assert_eq!(trap.trap_code(), Some(TrapCode::UnreachableCodeReached)); +} + +fn test_config() -> Config { + let mut config = Config::default(); + config + .wasmtime_config() + .wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable); + config +} + +async fn run_core_wasi_test<'a>( + args: impl IntoIterator, + f: impl FnOnce(&mut StoreBuilder), +) -> anyhow::Result { + let mut engine_builder = Engine::builder(&test_config()).unwrap(); + engine_builder + .add_host_component(MultiplierHostComponent) + .unwrap(); + let engine: Engine<()> = engine_builder.build(); + run_core_wasi_test_engine(&engine, args, f).await +} + +async fn run_core_wasi_test_engine<'a>( + engine: &Engine<()>, + args: impl IntoIterator, + f: impl FnOnce(&mut StoreBuilder), +) -> anyhow::Result { + let mut store_builder: StoreBuilder = engine.store_builder(); + let mut stdout_buf = store_builder.stdout_buffered(); + store_builder.stderr_pipe(TestWriter); + store_builder.args(args).unwrap(); + + f(&mut store_builder); + + let mut store = store_builder.build().unwrap(); + let module_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../target/test-programs/core-wasi-test.wasm"); + let module = Module::from_file(engine.as_ref(), module_path).unwrap(); + let instance_pre = engine.instantiate_pre(&module).unwrap(); + let instance = instance_pre.instantiate_async(&mut store).await.unwrap(); + let func = instance.get_func(&mut store, "_start").unwrap(); + + func.call_async(&mut store, &[], &mut []).await?; + + let stdout = String::from_utf8(stdout_buf.take())?.trim_end().into(); + Ok(stdout) +} + +// Simple test HostComponent; multiplies the input by the configured factor +#[derive(Clone)] +struct MultiplierHostComponent; + +impl HostComponent for MultiplierHostComponent { + type Data = i32; + + fn add_to_linker( + linker: &mut spin_core::Linker, + get: impl Fn(&mut spin_core::Data) -> &mut Self::Data + Send + Sync + Copy + 'static, + ) -> anyhow::Result<()> { + // NOTE: we're trying to avoid wit-bindgen because a git dependency + // would make this crate unpublishable on crates.io + linker.func_wrap1_async("multiplier", "multiply", move |mut caller, input: i32| { + Box::new(async move { + let &mut factor = get(caller.data_mut()); + let output = factor * input; + Ok(output) + }) + })?; + Ok(()) + } + + fn build_data(&self) -> Self::Data { + 2 + } +} + +// Write with `print!`, required for test output capture +struct TestWriter; + +impl std::io::Write for TestWriter { + fn write(&mut self, buf: &[u8]) -> std::io::Result { + print!("{}", String::from_utf8_lossy(buf)); + Ok(buf.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } +} From c12caa55037086185cdebeef8406cca09f683c71 Mon Sep 17 00:00:00 2001 From: Lann Martin Date: Tue, 27 Sep 2022 12:49:45 -0400 Subject: [PATCH 28/28] Fix `--temp` The new trigger code expects the working directory to be absolute, which is the case for tempdirs but not necessarily `--temp`. Fix by `.canonicalize()`ing the work dir. Signed-off-by: Lann Martin --- src/commands/up.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/commands/up.rs b/src/commands/up.rs index 5ffc32c50..958a59ccd 100644 --- a/src/commands/up.rs +++ b/src/commands/up.rs @@ -119,7 +119,7 @@ impl UpCommand { None => WorkingDirectory::Temporary(tempfile::tempdir()?), Some(d) => WorkingDirectory::Given(d.to_owned()), }; - let working_dir = working_dir_holder.path(); + let working_dir = working_dir_holder.path().canonicalize()?; let mut app = match (&self.app, &self.bindle) { (app, None) => { @@ -127,10 +127,10 @@ impl UpCommand { .as_deref() .unwrap_or_else(|| DEFAULT_MANIFEST_FILE.as_ref()); let bindle_connection = self.bindle_connection(); - spin_loader::from_file(manifest_file, working_dir, &bindle_connection).await? + spin_loader::from_file(manifest_file, &working_dir, &bindle_connection).await? } (None, Some(bindle)) => match &self.server { - Some(server) => spin_loader::from_bindle(bindle, server, working_dir).await?, + Some(server) => spin_loader::from_bindle(bindle, server, &working_dir).await?, _ => bail!("Loading from a bindle requires a Bindle server URL"), }, (Some(_), Some(_)) => bail!("Specify only one of app file or bindle ID"), @@ -149,7 +149,7 @@ impl UpCommand { }; // Build and write app lock file - let locked_app = spin_trigger::locked::build_locked_app(app, working_dir)?; + let locked_app = spin_trigger::locked::build_locked_app(app, &working_dir)?; let locked_path = working_dir.join("spin.lock"); let locked_app_contents = serde_json::to_vec_pretty(&locked_app).context("failed to serialize locked app")?;