feat: implement package manager

Cargo.toml

@@ -16,18 +16,20 @@ members = [
   "rsquickjs",
 
   # xmas-js
+  "package-manager",
   "modules",
   "repl",
 ]
 
 [workspace.dependencies]
 # TODO: modify quickjs and rsquickjs-sys later
 rquickjs-sys = { version = "0.10.0", git = "https://github.com/DelSkayn/rquickjs", rev = "111951b1a31075bdff46684e0ff29f37ff2a04b2" }
-
 rsquickjs-core = { version = "0.10.0", path = "rsquickjs/core" }
 rsquickjs-macro = { version = "0.10.0", path = "rsquickjs/macro" }
 rsquickjs = { version = "0.10.0", path = "rsquickjs" }
+
 xmas-js-modules = { path = "modules" }
+xmas-package-manager = { path = "package-manager" }
 
 [dependencies]
 xmas-js-modules = { workspace = true }

README.md

@@ -207,6 +207,7 @@ Xmas.JS stands on the shoulders of giants:
 - **[rquickjs](https://github.com/DelSkayn/rquickjs)** - Rust bindings (we maintain a fork)
 - **[LLRT](https://github.com/awslabs/llrt)** - Inspiration and code for AWS Lambda optimization
 - **[Tokio](https://tokio.rs/)** - Async runtime that powers our I/O
+- **[Cotton](https://github.com/danielhuang/cotton)** - Package manager forked for our needs
 
 **Inspired by:**
 - [Deno](https://deno.land/) - Modern JavaScript runtime design

TODO.md

@@ -4,9 +4,6 @@ i decided to use WAMR,
 because it is more lightweight and mature than other runtimes like wasmtime or wasmer.
 in addition, WAMR has GC support and both JIT and AOT compilation modes.
 
-# shell for executing package.json scripts
-
-`deno_task_shell` is a great example
 
 # Networking API
 
@@ -32,13 +29,6 @@ and the host application can grant or deny those permissions.
 
 for both main thread and worker threads.
 
-# Package Manager
-
-i want to implement a fast and lightweight package manager,
-maybe i need to research at [cotton](https://github.com/danielhuang/cotton) which
-does not currently support Git repositories or local paths as dependencies;
-and i also want to support jsr and http package sources.
-
 # bundler / minifier / optimizer / typescript / lint / etc...
 
 OXC is great!
@@ -49,12 +39,11 @@ xmas: start repl
 - [ ] run <file>.<js/ts/mjs/cjs/jsx/tsx>
 - [ ] run also could behave like npx if not running a file
 
-- [ ] add : in project add to project.json dependencies, else add to global cache
-- [ ] remove : in project remove from project.json dependencies, else remove from global cache
-- [ ] install i : install all dependencies in project.json
-- [ ] update u : update all dependencies in project.json
-- [ ] list ls : list all dependencies in project.json, or global cache if not in project
-- [ ] execute exec <script> [args...] : execute script from project.json scripts
+- [x] add : in project add to project.json dependencies, else add to global cache
+- [x] install i : install all dependencies in project.json
+- [x] update u : update all dependencies in project.json
+- [x] list ls : list all dependencies in project.json, or global cache if not in project
+- [x] execute exec <script> [args...] : execute script from project.json scripts
 
 - [ ] compile <input> <output> (compile to quickjs bytecode)
 - [ ] init (project): create a new xmas project
@@ -107,12 +96,7 @@ i want to implement those API's as well.
 - [ ] WritableStream
 - [ ] WritableStreamDefaultController
 
-- [ ] WASM
-    - [ ] WebAssembly.Global
-    - [ ] WebAssembly.Instance
-    - [ ] WebAssembly.Memory
-    - [ ] WebAssembly.Module
-    - [ ] WebAssembly.Table
+
 
 Global methods / properties:
 - [x] globalThis
@@ -134,9 +118,30 @@ Global methods / properties:
 - [x] globalThis.setInterval() / globalThis.clearInterval()
 - [x] globalThis.structuredClone()
 
+---
+
 - [ ] WASM
+    - [ ] WebAssembly.Global
+    - [ ] WebAssembly.Instance
+    - [ ] WebAssembly.Memory
+    - [ ] WebAssembly.Module
+    - [ ] WebAssembly.Table
     - [ ] globalThis.WebAssembly.compile()
     - [ ] globalThis.WebAssembly.compileStreaming()
     - [ ] globalThis.WebAssembly.instantiate()
     - [ ] globalThis.WebAssembly.instantiateStreaming()
-    - [ ] globalThis.WebAssembly.validate()
+    - [ ] globalThis.WebAssembly.validate()
+
+---
+
+- [ ] globalThis
+  - [ ] $ for shell commands
+  - [ ] cli
+    - [ ] prompt
+    - [ ] ansi
+    - [ ] command
+
+- [x] Repl
+  - [ ] complete method/property name
+  - [ ] complete globalThis property name
+  - [x] package manager commands

package-manager/Cargo.toml

@@ -0,0 +1,57 @@
+[package]
+name = "xmas-package-manager"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+tracing = "0.1.40"
+
+tokio = { version = "1.37.0", features = ["full"] }
+tokio-tar = { version = "*" }
+tokio-util = { version = "0.7.10", features = ["compat"] }
+
+itertools = "0.14.0"
+
+
+async-compression = { version = "0.4.9", features = ["tokio", "gzip"] }
+async-recursion = "1.1.1"
+cached = { version = "0.56.0", features = ["async"] }
+
+clap = { version = "4.5.4", features = ["derive"] }
+color-eyre = "0.6.3"
+compact_str = { version = "0.9.0", features = ["serde"] }
+dashmap = { version = "6.0.0", features = ["serde"] }
+async-channel = "2.5.0"
+futures = "0.3.31"
+indexmap = { version = "2.2.6", features = ["serde"] }
+indicatif = "0.18.0"
+multimap = "0.10.0"
+node-semver = { git = "https://github.com/danielhuang/node-semver-rs", rev = "bf4b103dc88b310c9dc049433aff1a14716e1e68" }
+notify = "=8.2.0"
+reqwest = { version = "0.12.4", features = [
+    "json",
+    "stream",
+    "rustls-tls",
+    "trust-dns",
+    "brotli",
+    "gzip",
+    "deflate",
+    "http2",
+], default-features = false }
+
+rustc-hash = "2.0.0"
+
+serde = { version = "1.0.200", features = ["derive", "rc"] }
+serde_json = { version = "1.0.116", features = ["preserve_order"] }
+serde_path_to_error = "0.1.16"
+
+
+toml = "0.9.0"
+tap = "1.0.1"
+url = { version = "2.5.0", features = ["serde"] }
+rand = "0.8.5"
+which = "8.0.0"
+deno_task_shell = "0.26.1"
+owo-colors = "4.2.3"
+junction = "1.3.0"
+exec = "0.3.1"

package-manager/src/cache.rs

@@ -0,0 +1,53 @@
+use std::{fmt::Debug, hash::Hash, sync::Arc};
+
+use dashmap::DashMap;
+use futures::{
+    future::{BoxFuture, Shared},
+    Future, FutureExt,
+};
+
+use crate::progress::PROGRESS_BAR;
+
+type SharedBoxFuture<T> = Shared<BoxFuture<'static, T>>;
+
+pub struct Cache<K: Eq + Hash + Clone + Send + Debug + 'static, V: Clone + Send + 'static> {
+    loader: Box<dyn Fn(K) -> BoxFuture<'static, V> + Send + Sync + 'static>,
+    map: DashMap<K, SharedBoxFuture<V>>,
+}
+
+impl<K: Eq + Hash + Clone + Send + Debug + 'static, V: Clone + Send + 'static> Cache<K, V> {
+    pub fn new<T, F>(loader: T) -> Self
+    where
+        F: Future<Output = V> + Sized + Send + 'static,
+        T: Fn(K) -> F + Send + Sync + Clone + 'static,
+    {
+        let loader = Arc::new(loader);
+
+        Self {
+            loader: Box::new({
+                move |key| {
+                    let loader = loader.clone();
+                    Box::pin({
+                        async move {
+                            PROGRESS_BAR.inc_length(1);
+                            let v = loader(key).await;
+                            PROGRESS_BAR.inc(1);
+                            v
+                        }
+                    })
+                }
+            }),
+            map: DashMap::new(),
+        }
+    }
+
+    pub async fn get(&self, key: K) -> V {
+        let f = self
+            .map
+            .entry(key.clone())
+            .or_insert_with(|| (self.loader)(key).boxed().shared())
+            .clone();
+
+        f.await
+    }
+}

package-manager/src/cli.rs

@@ -0,0 +1,78 @@
+//! Command-line interface definitions for Cotton.
+
+use clap::Parser;
+use compact_str::CompactString;
+use node_semver::Version;
+use std::ffi::OsString;
+use std::path::PathBuf;
+
+#[derive(Parser, Debug)]
+#[clap(author, version, about, long_about = None)]
+pub struct Args {
+    /// Print verbose logs (including progress indicators)
+    #[clap(short, long, global = true)]
+    pub verbose: bool,
+    /// Prevent any modifications to the lockfile
+    #[clap(long, global = true)]
+    pub immutable: bool,
+    /// Run in a custom working directory
+    #[clap(long, global = true, alias = "cwd")]
+    pub working_dir: Option<PathBuf>,
+
+    /// Subcommand to execute
+    #[clap(subcommand)]
+    pub cmd: Subcommand,
+}
+
+#[derive(Parser, Debug, Clone)]
+pub enum Subcommand {
+    /// Install packages defined in package.json
+    #[clap(alias = "i")]
+    Install,
+    /// Prepare and save a newly planned lockfile
+    Update,
+    /// Add package to package.json
+    #[clap(alias = "a")]
+    Add {
+        names: Vec<CompactString>,
+        /// Add to `devDependencies` instead of `dependencies`
+        #[clap(short = 'D', long)]
+        dev: bool,
+        /// Pin dependencies to a specific version
+        #[clap(long, alias = "exact")]
+        pin: bool,
+    },
+    /// Run a script defined in package.json
+    Run {
+        name: CompactString,
+        #[clap(long)]
+        watch: Vec<PathBuf>,
+    },
+    /// Clean packages installed in `node_modules` and remove cache
+    Clean,
+    /// Update packages specified in package.json to the latest available version
+    Upgrade {
+        /// Pin dependencies to a specific version
+        #[clap(long)]
+        pin: bool,
+    },
+    /// Execute a command that is not specified as a script
+    Exec { exe: OsString, args: Vec<OsString> },
+    /// Remove package from package.json
+    Remove {
+        names: Vec<CompactString>,
+        /// Remove from `devDependencies` instead of `dependencies`
+        #[clap(short = 'D', long)]
+        dev: bool,
+    },
+    /// Find all uses of a given package
+    Why {
+        name: CompactString,
+        version: Option<Version>,
+    },
+    /// Create new projects from a `create-` starter kit
+    Create { name: CompactString },
+    /// Download (if needed) and execute a command
+    #[clap(name = "x")]
+    DownloadAndExec { name: OsString, args: Vec<OsString> },
+}

package-manager/src/commands/add.rs

@@ -0,0 +1,68 @@
+//! Add command implementation.
+
+use color_eyre::eyre::{ContextCompat, Result};
+use color_eyre::owo_colors::OwoColorize;
+use compact_str::CompactString;
+use futures::future::try_join_all;
+use serde_json::Value;
+
+use crate::npm::fetch_package;
+use crate::progress::{log_progress, PROGRESS_BAR};
+use crate::util::{read_package_or_default, save_package};
+
+/// Execute the add command.
+pub async fn cmd_add(names: &[CompactString], dev: bool, pin: bool) -> Result<()> {
+    if names.is_empty() {
+        PROGRESS_BAR.suspend(|| println!("Note: no packages specified"));
+    }
+
+    add_packages(names, dev, pin).await
+}
+
+/// Add packages to package.json.
+pub async fn add_packages(names: &[CompactString], dev: bool, pin: bool) -> Result<()> {
+    let mut package: Value = read_package_or_default().await?;
+    let dependencies = package
+        .as_object_mut()
+        .wrap_err("`package.json` is invalid")?
+        .entry(if dev {
+            "devDependencies"
+        } else {
+            "dependencies"
+        })
+        .or_insert(Value::Object(Default::default()))
+        .as_object_mut()
+        .wrap_err("`package.json` contains non-object dependencies field")?;
+
+    PROGRESS_BAR.set_message("Resolving packages".to_string());
+    PROGRESS_BAR.set_length(names.len() as u64);
+
+    for (name, res) in try_join_all(names.iter().map(|name| async move {
+        let x = fetch_package(name).await.map(|res| (name, res));
+        PROGRESS_BAR.inc(1);
+        PROGRESS_BAR.set_message(format!("Resolved {name}"));
+        x
+    }))
+    .await?
+    {
+        let latest = res
+            .dist_tags
+            .get("latest")
+            .wrap_err("Package `latest` tag not specified")?;
+
+        let version = if pin {
+            latest.to_string()
+        } else {
+            format!("^{latest}")
+        };
+
+        dependencies.insert(name.to_string(), Value::String(version.to_string()));
+
+        PROGRESS_BAR.suspend(|| println!("Added {} {}", name.yellow(), version.yellow()));
+    }
+
+    PROGRESS_BAR.finish_and_clear();
+    save_package(&package).await?;
+
+    Ok(())
+}

package-manager/src/commands/clean.rs

@@ -0,0 +1,17 @@
+//! Clean command implementation.
+
+use color_eyre::eyre::Result;
+use std::fs::remove_dir_all;
+use std::io::ErrorKind;
+
+/// Execute the clean command.
+pub fn cmd_clean() -> Result<()> {
+    for dir in ["node_modules", ".xmas"] {
+        match remove_dir_all(dir) {
+            Ok(()) => {}
+            Err(e) if e.kind() == ErrorKind::NotFound => {}
+            r => r?,
+        }
+    }
+    Ok(())
+}