diff --git a/Cargo.lock b/Cargo.lock index eccc775..d566c54 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -486,7 +486,7 @@ dependencies = [ [[package]] name = "componentize-py" -version = "0.22.1" +version = "0.23.0" dependencies = [ "anyhow", "assert_cmd", diff --git a/Cargo.toml b/Cargo.toml index 7a0f92f..2440d73 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "componentize-py" -version = "0.22.1" +version = "0.23.0" edition = "2024" exclude = ["cpython"] diff --git a/examples/cli-p3/README.md b/examples/cli-p3/README.md index b3ee3e4..2f8e3d8 100644 --- a/examples/cli-p3/README.md +++ b/examples/cli-p3/README.md @@ -11,7 +11,7 @@ run a Python-based component targetting version `0.3.0-rc-2026-03-15` of the ## Prerequisites * `Wasmtime` 43.0.0 -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -19,7 +19,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v43.0.0. ``` cargo install --version 43.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 ``` ## Running the demo diff --git a/examples/cli/README.md b/examples/cli/README.md index 30f77d2..b48ef91 100644 --- a/examples/cli/README.md +++ b/examples/cli/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-cli] `command` world. ## Prerequisites * `Wasmtime` 38.0.0 or later -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v38.0.0. ``` cargo install --version 38.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 ``` ## Running the demo diff --git a/examples/http-p3/README.md b/examples/http-p3/README.md index 5d4f37d..f61e1bc 100644 --- a/examples/http-p3/README.md +++ b/examples/http-p3/README.md @@ -11,7 +11,7 @@ run a Python-based component targetting version `0.3.0-rc-2026-03-15` of the ## Prerequisites * `Wasmtime` 43.0.0 -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -19,7 +19,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v43.0.0. ``` cargo install --version 43.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 ``` ## Running the demo diff --git a/examples/http/README.md b/examples/http/README.md index 042c27d..cbd9d90 100644 --- a/examples/http/README.md +++ b/examples/http/README.md @@ -10,7 +10,7 @@ run a Python-based component targetting the [wasi-http] `proxy` world. ## Prerequisites * `Wasmtime` 38.0.0 or later -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -18,7 +18,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v38.0.0. ``` cargo install --version 38.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 ``` ## Running the demo diff --git a/examples/matrix-math/README.md b/examples/matrix-math/README.md index 7c46497..3be2802 100644 --- a/examples/matrix-math/README.md +++ b/examples/matrix-math/README.md @@ -11,7 +11,7 @@ within a guest component. ## Prerequisites * `wasmtime` 38.0.0 or later -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 * `NumPy`, built for WASI Note that we use an unofficial build of NumPy since the upstream project does @@ -23,7 +23,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v38.0.0. ``` cargo install --version 38.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 curl -OL https://github.com/dicej/wasi-wheels/releases/download/v0.0.2/numpy-wasi.tar.gz tar xf numpy-wasi.tar.gz ``` diff --git a/examples/sandbox/README.md b/examples/sandbox/README.md index ac46056..93c52df 100644 --- a/examples/sandbox/README.md +++ b/examples/sandbox/README.md @@ -12,10 +12,10 @@ versions have a different API for working with components, and this example has not yet been updated to use it. * `wasmtime-py` 38.0.0 -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 ``` -pip install componentize-py==0.22.1 wasmtime==38.0.0 +pip install componentize-py==0.23.0 wasmtime==38.0.0 ``` ## Running the demo diff --git a/examples/tcp-p3/README.md b/examples/tcp-p3/README.md index bd7fb9b..5347d7d 100644 --- a/examples/tcp-p3/README.md +++ b/examples/tcp-p3/README.md @@ -12,7 +12,7 @@ run a Python-based component targetting version `0.3.0-rc-2026-03-15` of the ## Prerequisites * `Wasmtime` 43.0.0 -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -20,7 +20,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v43.0.0. ``` cargo install --version 43.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 ``` ## Running the demo diff --git a/examples/tcp/README.md b/examples/tcp/README.md index 6eea088..5dca29c 100644 --- a/examples/tcp/README.md +++ b/examples/tcp/README.md @@ -11,7 +11,7 @@ making an outbound TCP request using `wasi-sockets`. ## Prerequisites * `Wasmtime` 38.0.0 or later -* `componentize-py` 0.22.1 +* `componentize-py` 0.23.0 Below, we use [Rust](https://rustup.rs/)'s `cargo` to install `Wasmtime`. If you don't have `cargo`, you can download and install from @@ -19,7 +19,7 @@ https://github.com/bytecodealliance/wasmtime/releases/tag/v38.0.0. ``` cargo install --version 38.0.0 wasmtime-cli -pip install componentize-py==0.22.1 +pip install componentize-py==0.23.0 ``` ## Running the demo diff --git a/pyproject.toml b/pyproject.toml index c8429f6..c2bb0c2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -7,7 +7,7 @@ features = ["pyo3/extension-module"] [project] name = "componentize-py" -version = "0.22.1" +version = "0.23.0" description = "Tool to package Python applications as WebAssembly components" readme = "README.md" license = { file = "LICENSE" } diff --git a/src/command.rs b/src/command.rs index e519cab..d206abe 100644 --- a/src/command.rs +++ b/src/command.rs @@ -102,6 +102,14 @@ pub enum Command { /// Generate Python bindings for the world and write them to the specified /// directory. + /// + /// Note that bindings generated using this command are only stubs which + /// unconditionally raise `NotImplementedError`s. They are meant for use by + /// e.g. IDEs, type checkers, and document generators. The `componentize` + /// subcommand will generate the real code (i.e. code connected to the + /// components actual imports and exports) on-the-fly as necessary. The + /// bindings generated by this command need not and should not be used by an + /// actual component produced using the `componentize` command. Bindings(Bindings), } @@ -183,7 +191,7 @@ pub fn run + Clone, I: IntoIterator>(args: I) -> Res fn generate_bindings(common: Common, bindings: Bindings) -> Result<()> { BindingsGenerator { - wit_path: &common + wit_paths: &common .wit_path .iter() .map(|v| v.as_path()) @@ -226,7 +234,7 @@ fn componentize(common: Common, componentize: Componentize) -> Result<()> { Runtime::new()?.block_on( ComponentGenerator { - wit_path: &common + wit_paths: &common .wit_path .iter() .map(|v| v.as_path()) @@ -521,4 +529,117 @@ class Bindings(bindings.Bindings): }; componentize(common, componentize_opts) } + + #[test] + fn mix_of_cli_and_dependency_wits_and_worlds() -> Result<()> { + let dir = tempfile::tempdir()?; + + let cli_wit_file = dir.path().join("cli.wit"); + fs::write( + &cli_wit_file, + br#" +package test:cli; + +interface cli-interface { + foo: func(); +} + +world cli-world { + import cli-interface; + export cli-interface; + export foo: func(); +} +"#, + )?; + + let app_file = dir.path().join("app.py"); + fs::write( + &app_file, + br#" +import cli_world +from cli_world import exports +from cli_world.imports import cli_interface +from lib.wit.imports import lib_interface + +class CliWorld(cli_world.CliWorld): + def foo(self) -> None: + pass + +class CliInterface(exports.CliInterface): + def foo(self) -> None: + lib_interface.foo() + cli_interface.foo() +"#, + )?; + + let lib_dir = dir.path().join("lib"); + let lib_wit_dir = lib_dir.join("wit"); + fs::create_dir_all(&lib_wit_dir)?; + + fs::write( + lib_dir.join("componentize-py.toml"), + br#" +wit_directory = "wit" +bindings = "wit" +"#, + )?; + + fs::write( + lib_wit_dir.join("lib.wit"), + br#" +package test:lib; + +interface lib-interface { + foo: func(); +} + +world lib-world { + import lib-interface; +} +"#, + )?; + + generate_bindings( + Common { + wit_path: vec![lib_wit_dir.clone()], + world: vec!["test:lib/lib-world".into()], + world_module: Some("lib.wit".into()), + quiet: false, + features: Vec::new(), + all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), + full_names: false, + }, + Bindings { + output_dir: lib_wit_dir, + }, + )?; + + componentize( + Common { + wit_path: vec![cli_wit_file], + world: vec!["test:cli/cli-world".into(), "test:lib/lib-world".into()], + world_module: Some("cli_world".into()), + quiet: false, + features: Vec::new(), + all_features: false, + import_interface_name: Vec::new(), + export_interface_name: Vec::new(), + full_names: false, + }, + Componentize { + app_name: "app".into(), + python_path: vec![ + dir.path() + .to_str() + .ok_or_else(|| anyhow::anyhow!("non-UTF8 path"))? + .into(), + ], + module_worlds: vec![], + output: dir.path().join("app.wasm"), + stub_wasi: false, + }, + ) + } } diff --git a/src/lib.rs b/src/lib.rs index a479057..25e5586 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,7 @@ #![deny(warnings)] use { - anyhow::{Context, Error, Result, anyhow, bail, ensure}, + anyhow::{Context, Error, Result, anyhow, ensure}, async_trait::async_trait, bytes::Bytes, component_init_transform::Invoker, @@ -32,8 +32,8 @@ use { wit_component::metadata, wit_dylib::DylibOpts, wit_parser::{ - CloneMaps, FunctionKind, Package, PackageName, Resolve, Stability, TypeDefKind, World, - WorldId, WorldItem, WorldKey, + CloneMaps, FunctionKind, Package, PackageId, PackageName, Resolve, Stability, TypeDefKind, + World, WorldId, WorldItem, WorldKey, }, zstd::Decoder, }; @@ -178,7 +178,7 @@ impl Invoker for MyInvoker { } pub struct BindingsGenerator<'a> { - pub wit_path: &'a [&'a Path], + pub wit_paths: &'a [&'a Path], pub worlds: &'a [&'a str], pub features: &'a [&'a str], pub all_features: bool, @@ -191,17 +191,44 @@ pub struct BindingsGenerator<'a> { impl BindingsGenerator<'_> { pub fn generate(&self) -> Result<()> { - // TODO: Split out and reuse the code responsible for finding and using - // componentize-py.toml files in the `componentize` function below, since - // that can affect the bindings we should be generating. + // Here we parse the specified WIT paths and resolve the specified + // worlds, then union them together and generate stub code for the + // unioned world. + // + // Note that this code is not meant to be run; every import raises a + // `NotImplementedError`. This is only meant for use by e.g. IDEs, type + // checkers, and document generators. The real code (i.e. the code + // which is actually hooked up to real component imports and exports) + // will be generated on-the-fly by `ComponentGenerator::componentize`. + // + // Note that, unlike in `ComponentGenerator::componentize`, we make no + // attempt to discover or use `componentize-py.toml` files here; we only + // use the paths in `self.wit_paths` and only output the generated code + // to a single directory. + + let mut resolve = Resolve { + all_features: self.all_features, + features: parse_features(self.features), + ..Default::default() + }; + + let mut packages = Vec::new(); + for &path in self.wit_paths { + packages.push((path, resolve.push_path(path)?.0)); + } + + if packages.is_empty() { + // If no WIT directory was provided as a parameter, use ./wit by default. + packages.push((Path::new("wit"), resolve.push_path("wit")?.0)); + } + + let worlds = select_worlds(&resolve, self.worlds, &packages)?; + let world = match &worlds.iter().copied().collect::>()[..] { + [] => select_world(&resolve, None, &packages)?, + &[world] => world, + worlds => union_world(&mut resolve, "union", worlds, &mut CloneMaps::default())?, + }; - let (resolve, world) = parse_wit( - self.wit_path, - self.worlds, - self.features, - self.all_features, - "union", - )?; let import_function_indexes = &HashMap::new(); let export_function_indexes = &HashMap::new(); let stream_and_future_indexes = &HashMap::new(); @@ -240,7 +267,7 @@ impl BindingsGenerator<'_> { pub type AddToLinker<'a> = Option<&'a dyn Fn(&mut Linker) -> Result<()>>; pub struct ComponentGenerator<'a> { - pub wit_path: &'a [&'a Path], + pub wit_paths: &'a [&'a Path], pub worlds: &'a [&'a str], pub features: &'a [&'a str], pub all_features: bool, @@ -258,6 +285,44 @@ pub struct ComponentGenerator<'a> { impl ComponentGenerator<'_> { pub async fn generate(&self) -> Result<()> { + // This is a _big_ function. Here's an overview of what it does. + // + // Our goal here is to collect one or more WIT paths, parse them, and + // resolve one or more WIT worlds, then generate a component that + // targets the union of those worlds. + // + // What complicates this process is that some of the WIT paths my have + // been specified explicitly via `self.wit_paths`, but some of them will + // be implicitly added as we traverse `self.python_path` and discover + // any `componentize-py.toml` files. A Python module containing such a + // file may use it specify its own WIT path as well is where to place + // the generated code for any worlds which have been resolved in that + // WIT path. In this case we say the Python module "covers" or "owns" + // those worlds. Therefore, while the output component will target the + // union of all the worlds, the code generated for that unioned world + // may be spread across multiple modules according to which modules + // "cover" which parts of the unioned world. + // + // Note that, when a given Python module "covers" a set of worlds, it + // may contain its own, pregenerated bindings for use by IDEs, + // type-checkers, and document generators. Those bindings will be + // ignored here and replaced by code we generate on-the-fly and which is + // hooked up to the native Wasm imports and exports we'll be + // synthesizing. + // + // In a nutshell, our job here is to not only create a component which + // targets the union of the specified worlds, but to generate code and + // place it into one or more modules as directed by the configuration + // settings specified in `self` and any `componentize-py.toml` files + // discovered. + // + // Once we've done that, there's one final step: pre-initialize the + // component. This involves running the top level script specified by + // `self.app_name` in a controled environment where it has access to all + // the directories in `self.python_path` plus directories containing any + // generated code produced earlier. Assuming this step succeeds, we'll + // snapshot the result and emit the snapshot as the final output. + // Remove non-existent elements from `python_path` so we don't choke on them // later: let python_path = &self @@ -282,22 +347,6 @@ impl ComponentGenerator<'_> { name }; - // Next, iterate over all the WIT directories, merging them into a single - // `Resolve`, and matching Python packages to `WorldId`s. - let (mut resolve, mut main_world) = match self.wit_path { - [] => (None, None), - paths => { - let (resolve, world) = parse_wit( - paths, - self.worlds, - self.features, - self.all_features, - &next_union_name(), - )?; - (Some(resolve), Some(world)) - } - }; - let import_interface_names = self .import_interface_names .iter() @@ -324,85 +373,159 @@ impl ComponentGenerator<'_> { })) .collect(); + let features = parse_features(self.features); + + let mut resolve = Resolve { + all_features: self.all_features, + features: features.clone(), + ..Default::default() + }; + + let mut packages = Vec::new(); + let configs = configs .iter() - .map(|(module, (config, worlds))| { + .map(|(module, &(ref config, worlds))| { Ok(( module, - match (worlds, config.config.wit_directory.as_deref()) { - (_, Some(wit_path)) => { - let path = config.path.join(wit_path); - let paths = &[path.as_path()]; - let (my_resolve, mut world) = parse_wit( - paths, - worlds, - self.features, - self.all_features, - &next_union_name(), - )?; - - if let Some(resolve) = &mut resolve { - let remap = resolve.merge(my_resolve)?; - world = remap.worlds[world.index()].expect("missing world"); - } else { - resolve = Some(my_resolve); - } + if let Some(path) = config.config.wit_directory.as_deref() { + // The list of worlds we have here includes all the + // worlds which might possibly be covered by this + // particular Python module, but not all of them + // necessarily _are_ covered by it. Here we filter the + // list by creating a `Resolve` which _only_ includes + // the path specified in this module's + // `componentize-py.toml` file and looking up the world + // there. If there's a match, we'll keep it; otherwise, + // we toss it out and assume it will be (or has been) + // covered elsewhere. + + let mut tmp = Resolve { + all_features: self.all_features, + features: features.clone(), + ..Default::default() + }; + + let package = tmp.push_path(path)?.0; + + let worlds = worlds + .iter() + .filter_map(|world| { + select_world(&tmp, Some(world), &[(path, package)]).ok() + }) + .collect::>(); - (config, Some(world)) - } - ([], None) => (config, None), - (_, None) => { - bail!( - "no `wit-directory` specified in \ - `componentize-py.toml` for module `{module}`" - ); - } + let remap = resolve.merge(tmp)?; + + packages.push((path, remap.packages[package.index()])); + + let worlds = worlds + .into_iter() + .map(|v| remap.worlds[v.index()].unwrap()) + .collect(); + + (config, worlds) + } else { + (config, Vec::new()) }, )) }) .collect::>>()?; - let mut resolve = if let Some(resolve) = resolve { - resolve - } else { + for path in self.wit_paths { + packages.push((path, resolve.push_path(path)?.0)); + } + + if packages.is_empty() { // If no WIT directory was provided as a parameter and none were - // referenced by Python packages, use the default values. - let paths: &[&Path] = &[]; - let (my_resolve, world) = parse_wit( - paths, - self.worlds, - self.features, - self.all_features, - &next_union_name(), - ) - .context( - "no WIT files found; please specify the directory or file \ - containing the WIT world you wish to target", - )?; - main_world = Some(world); - my_resolve - }; + // referenced by Python packages, use ./wit by default. + packages.push((Path::new("wit"), resolve.push_path("wit")?.0)); + } - // Extract relevant metadata from the `Resolve` into a `Summary` instance, - // which we'll use to generate Wasm- and Python-level bindings. + let worlds = select_worlds(&resolve, self.worlds, &packages)?; - let worlds = configs - .values() - .filter_map(|(_, world)| *world) - .chain(main_world) + let mut all_worlds = worlds + .iter() + .copied() + .chain(configs.values().flat_map(|(_, v)| v.iter().copied())) .collect::>(); + if all_worlds.is_empty() { + // No worlds specified; pick the default one, if available: + all_worlds.insert(select_world(&resolve, None, &packages)?); + } + + // Now that we've parsed all known WIT files and resolved all relevant + // worlds, we collect them all into a single world, unioning them + // together if there's more than one. + // + // This unified world represents the target world of the component we're + // creating, but note that, because parts of that world may be covered + // by dependencies, we may need to split code generation across several + // Python modules, so we still need to keep track of the original list + // of worlds and which modules they belong to. + let mut clone_maps = CloneMaps::default(); - let union_world = union_world( + + let mut unioned = |resolve: &mut _, worlds: &[_]| { + anyhow::Ok(match worlds { + [] => None, + &[world] => Some(world), + worlds => Some(union_world( + resolve, + &next_union_name(), + worlds, + &mut clone_maps, + )?), + }) + }; + + let world = unioned( &mut resolve, - &next_union_name(), - &worlds.iter().copied().collect::>(), - &mut clone_maps, + &all_worlds.iter().copied().collect::>(), + )? + .unwrap(); + + // Determine which worlds are covered by which Python modules and, for + // each module, union the ones covered by the module into a single + // world. + + let mut worlds_to_generate = all_worlds.clone(); + + let configs = configs + .iter() + .map(|(module, (config, worlds))| { + // If a `bindings` config is specified for this module, we will + // generate code for these worlds separately, so remove them + // from `worlds_to_generate`. + if config.config.bindings.is_some() { + worlds_to_generate = worlds_to_generate + .difference(&worlds.iter().copied().collect::>()) + .copied() + .collect(); + } + + let world = unioned(&mut resolve, worlds)?; + + Ok((module, (config, world))) + }) + .collect::>>()?; + + // Here we union together any worlds not covered by any of the Python + // modules above. We'll generate code for this world in its own, + // synthesized module. + + let world_to_generate = unioned( + &mut resolve, + &worlds_to_generate.into_iter().collect::>(), )?; + // Extract relevant metadata from the `Resolve` into a `Summary` instance, + // which we'll use to generate Wasm- and Python-level bindings. + let (mut bindings, metadata) = wit_dylib::create_with_metadata( &resolve, - union_world, + world, Some(&mut DylibOpts { interpreter: Some("libcomponentize_py_runtime.so".into()), async_: Default::default(), @@ -413,7 +536,7 @@ impl ComponentGenerator<'_> { name: Cow::Borrowed("component-type:componentize-py-union"), data: Cow::Owned(metadata::encode( &resolve, - union_world, + world, wit_component::StringEncoding::UTF8, None, )?), @@ -463,7 +586,7 @@ impl ComponentGenerator<'_> { let summary = Summary::try_new( &resolve, - &worlds, + &all_worlds, &import_interface_names, &export_interface_names, &imported_function_indexes, @@ -547,26 +670,16 @@ impl ComponentGenerator<'_> { wasi.preopened_dir(path, index.to_string(), DirPerms::all(), FilePerms::all())?; } - // For each Python package with a `componentize-py.toml` file that specifies - // where generated bindings for that package should be placed, generate the - // bindings and place them as indicated. + // For each Python module with a `componentize-py.toml` file that + // specifies where generated bindings for that package should be placed, + // generate the bindings and place them as indicated. let mut world_dir_mounts = Vec::new(); let mut locations = Locations::default(); - let mut saw_main_world = false; - - for (config, world, binding_path) in configs - .values() - .filter_map(|(config, world)| Some((config, world, config.config.bindings.as_deref()?))) - { - if *world == main_world { - saw_main_world = true; - } - - let Some(world) = *world else { - bail!("please specify a world for module `{}`", config.module); - }; + for (config, world, binding_path) in configs.values().filter_map(|&(ref config, world)| { + Some((config, world?, config.config.bindings.as_deref()?)) + }) { let paths = python_path .iter() .enumerate() @@ -607,9 +720,9 @@ impl ComponentGenerator<'_> { )); } - // If the caller specified a world and we haven't already generated bindings - // for it above, do so now. - if let (Some(world), false) = (main_world, saw_main_world) { + // Here we generate code for any worlds not covered by any of the Python + // modules we visited above. + if let Some(world) = world_to_generate { let module = self.world_module.unwrap_or(DEFAULT_WORLD_MODULE); let world_dir = tempfile::tempdir()?; let module_path = world_dir.path().join(module); @@ -701,7 +814,7 @@ impl ComponentGenerator<'_> { async move { let component = &Component::new(&engine, instrumented)?; if !added_to_linker { - add_wasi_and_stubs(&resolve, &worlds, &mut linker)?; + add_wasi_and_stubs(&resolve, &all_worlds, &mut linker)?; } let pre = InitPre::new(linker.instantiate_pre(component)?)?; @@ -737,107 +850,78 @@ impl ComponentGenerator<'_> { } } -fn parse_wit( - paths: &[&Path], - worlds: &[&str], - features: &[&str], - all_features: bool, - union_name: &str, -) -> Result<(Resolve, WorldId)> { - // If no WIT directory was provided as a parameter and none were referenced - // by Python packages, use ./wit by default. - if paths.is_empty() { - let paths = &[Path::new("wit")]; - return parse_wit(paths, worlds, features, all_features, union_name); - } - debug_assert!(!paths.is_empty(), "The paths should not be empty"); - - let mut resolve = Resolve { - all_features, - ..Default::default() - }; - for features in features { - for feature in features - .split(',') - .flat_map(|s| s.split_whitespace()) - .filter(|f| !f.is_empty()) - { - resolve.features.insert(feature.to_string()); - } - } - - let packages = paths +fn parse_features(features: &[&str]) -> IndexSet { + features .iter() - .map(|path| { - // Consolidates if the same package is referenced in multiple worlds - let mut tmp = Resolve { - all_features, - features: resolve.features.clone(), - ..Default::default() - }; - let (pkg, _files) = tmp.push_path(path)?; - let consolidated = resolve.merge(tmp)?; - Ok(consolidated.packages[pkg.index()]) + .flat_map(|features| { + features + .split(',') + .flat_map(|s| s.split_whitespace()) + .filter(|f| !f.is_empty()) }) - .collect::>>()?; + .map(String::from) + .collect() +} - let available_worlds = |resolve: &Resolve| { - resolve - .worlds - .iter() - .map(|(_, world)| { - if let Some(package) = world.package { - let package = &resolve.packages[package].name; - let version = if let Some(version) = &package.version { - format!("@{version}") - } else { - String::new() - }; - let package_namespace = &package.namespace; - let package_name = &package.name; - let name = &world.name; - format!("{package_namespace}:{package_name}/{name}{version}") +fn available_worlds(resolve: &Resolve) -> Vec { + resolve + .worlds + .iter() + .map(|(_, world)| { + if let Some(package) = world.package { + let package = &resolve.packages[package].name; + let version = if let Some(version) = &package.version { + format!("@{version}") } else { - world.name.clone() - } - }) - .collect::>() - }; + String::new() + }; + let package_namespace = &package.namespace; + let package_name = &package.name; + let name = &world.name; + format!("{package_namespace}:{package_name}/{name}{version}") + } else { + world.name.clone() + } + }) + .collect() +} - let worlds = worlds - .iter() - .map(|world| { - packages - .iter() - .find_map(|&pkg| resolve.select_world(&[pkg], Some(world)).ok()) - .ok_or_else(|| { - let worlds = available_worlds(&resolve); - anyhow!( - "No world named `{world}` found in any of the loaded WIT packages.\n\ +fn select_world( + resolve: &Resolve, + world: Option<&str>, + packages: &[(&Path, PackageId)], +) -> Result { + // First, try looking in the top-level packages + resolve + .select_world(&packages.iter().map(|&(_, v)| v).collect::>(), world) + .or_else(|_| { + // That didn't work; now try _all_ known packages + resolve + .select_world( + &resolve.packages.iter().map(|(v, _)| v).collect::>(), + world, + ) + .with_context(|| { + let worlds = available_worlds(resolve); + let paths = packages.iter().map(|&(v, _)| v).collect::>(); + format!( + "Unable to resolve `{world:?}`.\n\ Available worlds: {worlds:#?}\n\ WIT paths: {paths:#?}" ) }) }) - .collect::>>()?; - - let world = match &worlds[..] { - [] => packages - .iter() - .find_map(|&pkg| resolve.select_world(&[pkg], None).ok()) - .ok_or_else(|| { - let worlds = available_worlds(&resolve); - anyhow!( - "No default world found in any of the loaded WIT packages.\n\ - Available worlds: {worlds:#?}\n\ - WIT paths: {paths:#?}" - ) - })?, - &[world] => world, - worlds => union_world(&mut resolve, union_name, worlds, &mut CloneMaps::default())?, - }; +} - Ok((resolve, world)) +fn select_worlds( + resolve: &Resolve, + worlds: &[&str], + packages: &[(&Path, PackageId)], +) -> Result> { + worlds + .iter() + .map(|world| select_world(resolve, Some(world), packages)) + .collect() } fn union_world( diff --git a/src/python.rs b/src/python.rs index 515a785..e7aeb5b 100644 --- a/src/python.rs +++ b/src/python.rs @@ -37,7 +37,7 @@ fn python_componentize( (|| { Runtime::new()?.block_on( ComponentGenerator { - wit_path: &wit_path.iter().map(|v| v.as_path()).collect::>(), + wit_paths: &wit_path.iter().map(|v| v.as_path()).collect::>(), worlds: &worlds.iter().map(|v| v.as_str()).collect::>(), features: &features.iter().map(|v| v.as_str()).collect::>(), all_features, @@ -86,7 +86,7 @@ fn python_generate_bindings( full_names: bool, ) -> PyResult<()> { BindingsGenerator { - wit_path: &wit_path.iter().map(|v| v.as_path()).collect::>(), + wit_paths: &wit_path.iter().map(|v| v.as_path()).collect::>(), worlds: &worlds.iter().map(|v| v.as_str()).collect::>(), features: &features.iter().map(|v| v.as_str()).collect::>(), all_features, diff --git a/src/test.rs b/src/test.rs index ed0d9af..b61c92c 100644 --- a/src/test.rs +++ b/src/test.rs @@ -45,6 +45,7 @@ static ENGINE: Lazy = Lazy::new(|| { #[allow(clippy::type_complexity)] async fn make_component( wit: &str, + worlds: &[&str], world_module: Option<&str>, guest_code: &[(&str, &str)], python_path: &[&str], @@ -61,8 +62,8 @@ async fn make_component( } ComponentGenerator { - wit_path: &[&tempdir.path().join("app.wit")], - worlds: &[], + wit_paths: &[&tempdir.path().join("app.wit")], + worlds, features: &[], all_features: false, world_module, @@ -123,6 +124,7 @@ struct Tester { impl Tester { fn new( wit: &str, + worlds: &[&str], world_module: Option<&str>, guest_code: &[(&str, &str)], python_path: &[&str], @@ -136,6 +138,7 @@ impl Tester { // mechanism when pre-initializing. let component = &Runtime::new()?.block_on(make_component( wit, + worlds, world_module, guest_code, python_path, diff --git a/src/test/echoes.rs b/src/test/echoes.rs index 9795586..01a9bbd 100644 --- a/src/test/echoes.rs +++ b/src/test/echoes.rs @@ -321,6 +321,7 @@ class Echoes(exports.Echoes): static TESTER: Lazy> = Lazy::new(|| { Tester::::new( include_str!("wit/echoes.wit"), + &[], Some("echoes_test"), GUEST_CODE, &[], diff --git a/src/test/tests.rs b/src/test/tests.rs index 0fdc614..0927651 100644 --- a/src/test/tests.rs +++ b/src/test/tests.rs @@ -164,10 +164,14 @@ impl super::Host for BarHost { static TESTER: Lazy> = Lazy::new(|| { Tester::::new( include_str!("wit/tests.wit"), + &["componentize-py:test/tests"], Some("tests"), GUEST_CODE, &["src/test"], - &[("foo_sdk", &["foo-world"]), ("bar_sdk", &["bar-world"])], + &[ + ("foo_sdk", &["foo:sdk/foo-world"]), + ("bar_sdk", &["bar:sdk/bar-world"]), + ], *SEED, ) .unwrap() diff --git a/test-generator/src/lib.rs b/test-generator/src/lib.rs index e52d1a4..a60423f 100644 --- a/test-generator/src/lib.rs +++ b/test-generator/src/lib.rs @@ -870,6 +870,7 @@ class EchoesGenerated(exports.EchoesGenerated): static TESTER: Lazy> = Lazy::new(|| {{ Tester::::new( include_str!({wit_path:?}), + &[], Some("echoes_generated_test"), GUEST_CODE, &[],