diff --git a/README.md b/README.md index 1f70d1e..9494e8e 100644 --- a/README.md +++ b/README.md @@ -68,8 +68,9 @@ export const main = task({ }); ``` + > [!NOTE] -> TypeScript/JavaScript projects require a task named `"main"` as the entrypoint. +> The runtime requires a task named `"main"` as the entrypoint. Python can define the main task itself, but it's recommended to set it manually as well. When you run `capsule run main.py` (or `main.ts`), your code is compiled into a WebAssembly module and executed in a dedicated sandbox to isolate tasks. diff --git a/crates/capsule-cli/Cargo.lock b/crates/capsule-cli/Cargo.lock index c31895e..c5405eb 100644 --- a/crates/capsule-cli/Cargo.lock +++ b/crates/capsule-cli/Cargo.lock @@ -331,6 +331,7 @@ dependencies = [ "humantime", "indicatif", "nanoid", + "rayon", "reqwest", "rusqlite", "rustpython-parser", @@ -341,6 +342,7 @@ dependencies = [ "swc_ecma_parser", "tokio", "toml 0.9.11+spec-1.1.0", + "walkdir", "wasmtime", "wasmtime-wasi", "wasmtime-wasi-http", @@ -2311,6 +2313,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3177,6 +3188,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" diff --git a/crates/capsule-cli/docs/README-pypi.md b/crates/capsule-cli/docs/README-pypi.md index 28135f4..49ca6f7 100644 --- a/crates/capsule-cli/docs/README-pypi.md +++ b/crates/capsule-cli/docs/README-pypi.md @@ -50,6 +50,10 @@ def analyze_data(dataset: list) -> dict: return {"processed": len(dataset), "status": "complete"} ``` +> [!NOTE] +> The runtime requires a task named `"main"` as the entrypoint. Python can define the main task itself, but it's recommended to set it manually. + + When you run `capsule run main.py`, your code is compiled into a WebAssembly module and executed in a dedicated sandbox. ## Documentation diff --git a/crates/capsule-cli/npm/README.md b/crates/capsule-cli/npm/README.md index d9b42a6..ee8883a 100644 --- a/crates/capsule-cli/npm/README.md +++ b/crates/capsule-cli/npm/README.md @@ -60,7 +60,6 @@ export const analyzeData = task({ return { processed: dataset.length, status: "complete" }; }); -// The "main" task is required as the entrypoint export const main = task({ name: "main", compute: "HIGH" @@ -69,6 +68,9 @@ export const main = task({ }); ``` +> [!NOTE] +> The runtime requires a task named `"main"` as the entrypoint. + When you run `capsule run main.ts`, your code is compiled into a WebAssembly module and executed in a dedicated sandbox. ## Documentation diff --git a/crates/capsule-cli/src/commands/run.rs b/crates/capsule-cli/src/commands/run.rs index 199a3a8..b334d61 100644 --- a/crates/capsule-cli/src/commands/run.rs +++ b/crates/capsule-cli/src/commands/run.rs @@ -142,11 +142,7 @@ pub async fn execute( let execution_policy = extract_main_execution_policy(compile_result.task_registry, &manifest.capsule_toml) - .unwrap_or_else(|| { - ExecutionPolicy::default() - .compute(Some(Compute::Custom(u64::MAX))) - .allowed_files(vec![".".to_string()]) - }); + .unwrap_or_else(|| ExecutionPolicy::default().compute(Some(Compute::Custom(u64::MAX)))); let runtime = Runtime::new(runtime_config, manifest.capsule_toml)?; diff --git a/crates/capsule-core/Cargo.lock b/crates/capsule-core/Cargo.lock index 4fa3355..aa64c46 100644 --- a/crates/capsule-core/Cargo.lock +++ b/crates/capsule-core/Cargo.lock @@ -311,6 +311,7 @@ dependencies = [ "humantime", "indicatif", "nanoid", + "rayon", "reqwest", "rusqlite", "rustfmt", @@ -322,6 +323,7 @@ dependencies = [ "swc_ecma_parser", "tokio", "toml 0.9.11+spec-1.1.0", + "walkdir", "wasmtime", "wasmtime-wasi", "wasmtime-wasi-http", @@ -2458,6 +2460,15 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -3417,6 +3428,16 @@ version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "want" version = "0.3.1" diff --git a/crates/capsule-core/Cargo.toml b/crates/capsule-core/Cargo.toml index 288d9ff..04a181a 100644 --- a/crates/capsule-core/Cargo.toml +++ b/crates/capsule-core/Cargo.toml @@ -29,6 +29,8 @@ chrono = "0.4" colored = "2.1" toml = "0.9.11" blake3 = "1" +rayon = "1.10" +walkdir = "2" rustpython-parser = "0.4" swc_ecma_parser = "33.0" diff --git a/crates/capsule-core/src/config/manifest.rs b/crates/capsule-core/src/config/manifest.rs index f1e6931..c00bfcb 100644 --- a/crates/capsule-core/src/config/manifest.rs +++ b/crates/capsule-core/src/config/manifest.rs @@ -1,4 +1,5 @@ use serde::Deserialize; +use std::fmt; use std::fs; use std::path::{Path, PathBuf}; @@ -10,8 +11,8 @@ pub enum ManifestError { ParseError(String), } -impl std::fmt::Display for ManifestError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for ManifestError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { ManifestError::FsError(e) => write!(f, "Filesystem error > {}", e), ManifestError::ParseError(e) => write!(f, "Parse error > {}", e), diff --git a/crates/capsule-core/src/wasm/compiler/javascript.rs b/crates/capsule-core/src/wasm/compiler/javascript.rs index 0e3167d..5257b72 100644 --- a/crates/capsule-core/src/wasm/compiler/javascript.rs +++ b/crates/capsule-core/src/wasm/compiler/javascript.rs @@ -1,10 +1,11 @@ use std::collections::HashMap; +use std::fmt; use std::fs; use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use crate::config::fingerprint::SourceFingerprint; -use crate::wasm::utilities::introspection::javascript; +use crate::wasm::utilities::introspection::scanner; use crate::wasm::utilities::wit_manager::WitManager; #[derive(Debug)] @@ -14,8 +15,8 @@ pub enum JavascriptWasmCompilerError { CompileFailed(String), } -impl std::fmt::Display for JavascriptWasmCompilerError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { +impl fmt::Display for JavascriptWasmCompilerError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { JavascriptWasmCompilerError::FsError(msg) => write!(f, "Filesystem error > {}", msg), JavascriptWasmCompilerError::CommandFailed(msg) => { @@ -317,9 +318,7 @@ export const taskRunner = exports; } pub fn introspect_task_registry(&self) -> Option> { - let source = fs::read_to_string(&self.source_path).ok()?; - let is_typescript = self.source_path.extension().is_some_and(|ext| ext == "ts"); - - javascript::extract_js_task_configs(&source, is_typescript) + let source_dir = self.source_path.parent()?; + scanner::scan_js_tasks(source_dir) } } diff --git a/crates/capsule-core/src/wasm/compiler/python.rs b/crates/capsule-core/src/wasm/compiler/python.rs index 157c33c..e218d87 100644 --- a/crates/capsule-core/src/wasm/compiler/python.rs +++ b/crates/capsule-core/src/wasm/compiler/python.rs @@ -6,7 +6,7 @@ use std::process::Command; use std::process::Stdio; use crate::config::fingerprint::SourceFingerprint; -use crate::wasm::utilities::introspection::python; +use crate::wasm::utilities::introspection::scanner; use crate::wasm::utilities::wit_manager::WitManager; pub enum PythonWasmCompilerError { @@ -308,8 +308,7 @@ from capsule.app import TaskRunner, exports } pub fn introspect_task_registry(&self) -> Option> { - let source = fs::read_to_string(&self.source_path).ok()?; - - python::extract_python_task_configs(&source) + let source_dir = self.source_path.parent()?; + scanner::scan_python_tasks(source_dir) } } diff --git a/crates/capsule-core/src/wasm/execution_policy.rs b/crates/capsule-core/src/wasm/execution_policy.rs index 56c2192..89be0a8 100644 --- a/crates/capsule-core/src/wasm/execution_policy.rs +++ b/crates/capsule-core/src/wasm/execution_policy.rs @@ -25,7 +25,7 @@ impl Compute { } } -#[derive(Clone, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct ExecutionPolicy { pub name: String, pub compute: Compute, diff --git a/crates/capsule-core/src/wasm/utilities/introspection/javascript.rs b/crates/capsule-core/src/wasm/utilities/introspection/javascript.rs index afb2166..176a0aa 100644 --- a/crates/capsule-core/src/wasm/utilities/introspection/javascript.rs +++ b/crates/capsule-core/src/wasm/utilities/introspection/javascript.rs @@ -27,7 +27,7 @@ pub fn extract_js_task_configs( let mut parser = Parser::new(syntax, StringInput::from(&*fm), None); let module = parser.parse_module().ok()?; - let mut tasks: HashMap = HashMap::new(); + let mut tasks = HashMap::new(); for item in &module.body { if let ModuleItem::Stmt(stmt) = item @@ -82,6 +82,7 @@ fn extract_task_call( .and_then(|v| v.as_str()) .unwrap_or(&var_name) .to_string(); + config.insert( "name".to_string(), serde_json::Value::String(task_name.clone()), @@ -132,6 +133,7 @@ fn extract_js_literal(expr: &Expr) -> Option { .filter_map(|e| e.as_ref()) .map(|e| extract_js_literal(&e.expr)) .collect(); + items.map(serde_json::Value::Array) } _ => None, diff --git a/crates/capsule-core/src/wasm/utilities/introspection/mod.rs b/crates/capsule-core/src/wasm/utilities/introspection/mod.rs index b49f333..ddece89 100644 --- a/crates/capsule-core/src/wasm/utilities/introspection/mod.rs +++ b/crates/capsule-core/src/wasm/utilities/introspection/mod.rs @@ -1,2 +1,3 @@ pub mod javascript; pub mod python; +pub mod scanner; diff --git a/crates/capsule-core/src/wasm/utilities/introspection/python.rs b/crates/capsule-core/src/wasm/utilities/introspection/python.rs index 5d88325..fd05ed5 100644 --- a/crates/capsule-core/src/wasm/utilities/introspection/python.rs +++ b/crates/capsule-core/src/wasm/utilities/introspection/python.rs @@ -34,7 +34,7 @@ fn extract_task_decorator( ) -> Option> { match decorator { ast::Expr::Name(name) if name.id.as_str() == "task" => { - let mut config: HashMap = HashMap::new(); + let mut config = HashMap::new(); config.insert( "name".to_string(), serde_json::Value::String(func_name.to_string()), @@ -58,7 +58,7 @@ fn extract_call_args( call: &ast::ExprCall, func_name: &str, ) -> Option> { - let mut config: HashMap = HashMap::new(); + let mut config = HashMap::new(); for keyword in &call.keywords { if let Some(arg_name) = &keyword.arg @@ -96,6 +96,7 @@ fn extract_literal(expr: &ast::Expr) -> Option { ast::Expr::List(list) => { let items: Option> = list.elts.iter().map(extract_literal).collect(); + items.map(serde_json::Value::Array) } _ => None, @@ -131,19 +132,6 @@ def main(): assert!(configs.contains_key("main")); } - #[test] - fn test_task_with_allowed_files() { - let source = r#" -@task(name="main", allowed_files=["./data", "./config"]) -def main(): - pass -"#; - let configs = extract_python_task_configs(source).unwrap(); - let allowed = configs["main"]["allowed_files"].as_array().unwrap(); - assert_eq!(allowed.len(), 2); - assert_eq!(allowed[0], "./data"); - } - #[test] fn test_no_task_decorator() { let source = r#" diff --git a/crates/capsule-core/src/wasm/utilities/introspection/scanner.rs b/crates/capsule-core/src/wasm/utilities/introspection/scanner.rs new file mode 100644 index 0000000..e20c804 --- /dev/null +++ b/crates/capsule-core/src/wasm/utilities/introspection/scanner.rs @@ -0,0 +1,203 @@ +use rayon::prelude::*; +use std::collections::HashMap; +use std::fs; +use std::path::Path; +use walkdir::WalkDir; + +use crate::wasm::utilities::introspection::javascript::extract_js_task_configs; +use crate::wasm::utilities::introspection::python::extract_python_task_configs; + +const IGNORED_DIRS: &[&str] = &[ + "node_modules", + "__pycache__", + ".git", + ".capsule", + "dist", + "build", + ".venv", + "venv", + "target", +]; + +pub fn scan_python_tasks(source_dir: &Path) -> Option> { + let files = collect_source_files(source_dir, &["py"]); + + if files.is_empty() { + return None; + } + + let all_tasks = files + .par_iter() + .filter_map(|path| { + let source = fs::read_to_string(path).ok()?; + extract_python_task_configs(&source) + }) + .reduce(HashMap::new, |mut acc, tasks| { + acc.extend(tasks); + acc + }); + + if all_tasks.is_empty() { + None + } else { + Some(all_tasks) + } +} + +pub fn scan_js_tasks(source_dir: &Path) -> Option> { + let files = collect_source_files(source_dir, &["js", "ts", "mjs", "mts"]); + + if files.is_empty() { + return None; + } + + let all_tasks = files + .par_iter() + .filter_map(|path| { + let source = fs::read_to_string(path).ok()?; + let is_typescript = path + .extension() + .is_some_and(|ext| ext == "ts" || ext == "mts"); + extract_js_task_configs(&source, is_typescript) + }) + .reduce(HashMap::new, |mut acc, tasks| { + acc.extend(tasks); + acc + }); + + if all_tasks.is_empty() { + None + } else { + Some(all_tasks) + } +} + +fn collect_source_files(source_dir: &Path, extensions: &[&str]) -> Vec { + WalkDir::new(source_dir) + .into_iter() + .filter_entry(|entry| { + if entry.file_type().is_dir() { + let name = entry.file_name().to_string_lossy(); + return !IGNORED_DIRS.iter().any(|ignored| name == *ignored); + } + true + }) + .filter_map(|entry| entry.ok()) + .filter(|entry| { + if !entry.file_type().is_file() { + return false; + } + entry + .path() + .extension() + .and_then(|ext| ext.to_str()) + .is_some_and(|ext| extensions.contains(&ext)) + }) + .map(|entry| entry.into_path()) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::fs; + + fn create_temp_dir() -> std::path::PathBuf { + let dir = std::env::temp_dir().join(format!("capsule_test_{}", nanoid::nanoid!(8))); + fs::create_dir_all(&dir).unwrap(); + dir + } + + #[test] + fn test_scan_python_multiple_files() { + let temp_dir = create_temp_dir(); + + fs::write( + temp_dir.join("main.py"), + r#" +@task(name="main") +def main(): + pass +"#, + ) + .unwrap(); + + let tasks_dir = temp_dir.join("tasks"); + fs::create_dir_all(&tasks_dir).unwrap(); + + fs::write( + tasks_dir.join("compute.py"), + r#" +@task(name="heavy_compute", compute="HIGH", ram="HIGH") +def heavy_compute(): + pass +"#, + ) + .unwrap(); + + fs::write( + tasks_dir.join("io_tasks.py"), + r#" +@task(name="fetch_data", timeout="30s") +def fetch_data(): + pass + +@task(name="save_data", allowed_files=["./data"]) +def save_data(): + pass +"#, + ) + .unwrap(); + + let tasks = scan_python_tasks(&temp_dir).unwrap(); + + assert_eq!(tasks.len(), 4); + assert!(tasks.contains_key("main")); + assert!(tasks.contains_key("heavy_compute")); + assert!(tasks.contains_key("fetch_data")); + assert!(tasks.contains_key("save_data")); + + assert_eq!(tasks["heavy_compute"]["compute"], "HIGH"); + assert_eq!(tasks["fetch_data"]["timeout"], "30s"); + assert_eq!( + tasks["save_data"]["allowed_files"], + serde_json::json!(["./data"]) + ); + + fs::remove_dir_all(&temp_dir).ok(); + } + + #[test] + fn test_scan_js_typescript_files() { + let temp_dir = create_temp_dir(); + + fs::write( + temp_dir.join("main.ts"), + r#" +export const main = task({ name: "main", compute: "HIGH" }, (): string => { + return "hello"; +}); +"#, + ) + .unwrap(); + + fs::write( + temp_dir.join("utils.js"), + r#" +const helper = task({ name: "helper" }, () => { + return 42; +}); +"#, + ) + .unwrap(); + + let tasks = scan_js_tasks(&temp_dir).unwrap(); + + assert_eq!(tasks.len(), 2); + assert!(tasks.contains_key("main")); + assert!(tasks.contains_key("helper")); + assert_eq!(tasks["main"]["compute"], "HIGH"); + + fs::remove_dir_all(&temp_dir).ok(); + } +} diff --git a/crates/capsule-sdk/javascript/README.md b/crates/capsule-sdk/javascript/README.md index 132c7f8..1d7725a 100644 --- a/crates/capsule-sdk/javascript/README.md +++ b/crates/capsule-sdk/javascript/README.md @@ -60,7 +60,6 @@ export const analyzeData = task({ return { processed: dataset.length, status: "complete" }; }); -// The "main" task is required as the entrypoint export const main = task({ name: "main", compute: "HIGH" @@ -69,6 +68,10 @@ export const main = task({ }); ``` +> [!NOTE] +> The runtime requires a task named `"main"` as the entrypoint. + + When you run `capsule run main.ts`, your code is compiled into a WebAssembly module and executed in a dedicated sandbox. ## Documentation diff --git a/crates/capsule-sdk/javascript/src/files.ts b/crates/capsule-sdk/javascript/src/files.ts index 35b8ad8..dadcc91 100644 --- a/crates/capsule-sdk/javascript/src/files.ts +++ b/crates/capsule-sdk/javascript/src/files.ts @@ -181,7 +181,7 @@ export async function list(path: string = "."): Promise { const entries: string[] = []; let entry; - while ((entry = stream.readDirectoryEntry()) !== undefined && entry !== null) { + while ((entry = stream.readDirectoryEntry()) && entry) { if (entry.name) { entries.push(entry.name); } diff --git a/crates/capsule-sdk/javascript/src/hostApi.ts b/crates/capsule-sdk/javascript/src/hostApi.ts index 44d70e1..e4fa297 100644 --- a/crates/capsule-sdk/javascript/src/hostApi.ts +++ b/crates/capsule-sdk/javascript/src/hostApi.ts @@ -49,7 +49,7 @@ export function callHost( args: any[], config: Record ): string { - if (checkWasm() && hostModule !== null) { + if (checkWasm() && hostModule) { try { const result = hostModule.scheduleTask( name, diff --git a/crates/capsule-sdk/javascript/src/task.ts b/crates/capsule-sdk/javascript/src/task.ts index fa0f3fa..2f2ed59 100644 --- a/crates/capsule-sdk/javascript/src/task.ts +++ b/crates/capsule-sdk/javascript/src/task.ts @@ -70,7 +70,7 @@ interface TaskResult { * @returns String format for host API, or undefined */ function normalizeTimeout(timeout?: string | number): string | undefined { - if (timeout === undefined) return undefined; + if (!timeout) return undefined; if (typeof timeout === "number") { return `${timeout}ms`; }