Rust Bindings, IO Adapters, CMake

.gitignore

@@ -9,6 +9,8 @@ context-transfer-engine/compressor/test/results
 context-transfer-engine/compressor/results
 .claude/worktrees/
 
+**/target
+
 # CMake generated files (only in build directories)
 # Build directories already ignored above, but add specific patterns for safety
 compile_commands.json

CMakeLists.txt

@@ -37,6 +37,33 @@ option(WRP_CORE_ENABLE_DOCKER_CI "Enable Docker-based integration tests (require
 option(BUILD_SHARED_LIBS "Build shared libraries (.dll/.so) instead of static ones (.lib/.a)" ON)
 option(WRP_CORE_STATIC_DEPS "Link external dependencies statically (for self-contained wheels)" OFF)
 
+#------------------------------------------------------------------------------
+# Rust Bindings Configuration
+#------------------------------------------------------------------------------
+option(WRP_CORE_ENABLE_RUST "Enable Rust bindings (requires Corrosion)" ON)
+if(WRP_CORE_ENABLE_RUST)
+    message(STATUS "Rust bindings enabled - configuring Corrosion")
+
+    # Ensure cmake module path is set before including AutoConfigureRust
+    list(APPEND CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/cmake")
+
+    # Auto-configure Rust paths BEFORE FetchContent_MakeAvailable(Corrosion)
+    # because Corrosion's FindRust.cmake runs immediately
+    include(AutoConfigureRust)
+
+    # Fetch Corrosion
+    include(FetchContent)
+    FetchContent_Declare(
+        Corrosion
+        GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
+        GIT_TAG v0.5.1
+        GIT_SHALLOW TRUE
+    )
+    FetchContent_MakeAvailable(Corrosion)
+
+    message(STATUS "Corrosion configured - Rust bindings available")
+endif()
+
 # Installation Prefix Configuration
 # - For Conda builds: Set CMAKE_INSTALL_PREFIX via -DCMAKE_INSTALL_PREFIX=$CONDA_PREFIX
 # - For custom installs: Set CMAKE_INSTALL_PREFIX to desired root directory
@@ -84,7 +111,7 @@ message(STATUS "  Install prefix:     ${CMAKE_INSTALL_PREFIX}")
 #------------------------------------------------------------------------------
 # Transport and Network Features
 #------------------------------------------------------------------------------
-option(WRP_CORE_ENABLE_MPI "Enable MPI support" OFF)
+option(WRP_CORE_ENABLE_MPI "Enable MPI support" ON)
 option(WRP_CORE_ENABLE_ZMQ "Enable ZeroMQ transport" ON)
 option(WRP_CORE_ENABLE_LIBFABRIC "Enable Libfabric transport" OFF)
 option(WRP_CORE_ENABLE_THALLIUM "Build tests which depend on thallium" OFF)
@@ -108,7 +135,7 @@ option(WRP_CORE_ENABLE_HDF5 "Enable HDF5 support in CAE" ON)
 #------------------------------------------------------------------------------
 # System and Runtime Features
 #------------------------------------------------------------------------------
-option(WRP_CORE_ENABLE_ELF "Enable ELF support (required for CTE adapters)" OFF)
+option(WRP_CORE_ENABLE_ELF "Enable ELF support (required for CTE adapters)" ON)
 option(WRP_CORE_ENABLE_OPENMP "Enable the use of OpenMP" OFF)
 option(WRP_CORE_ENABLE_CUDA "Enable CUDA support" OFF)
 option(WRP_CORE_ENABLE_ROCM "Enable ROCm support" OFF)
@@ -132,12 +159,13 @@ set(HSHM_LOG_LEVEL "1" CACHE STRING "Log level threshold (0=Debug, 1=Info, 2=War
 #------------------------------------------------------------------------------
 # CTE Adapter Options (require WRP_CORE_ENABLE_ELF=ON)
 option(WRP_CTE_ENABLE_POSIX_ADAPTER "Enable POSIX adapter" ON)
-option(WRP_CTE_ENABLE_STDIO_ADAPTER "Enable STDIO adapter" OFF)
-option(WRP_CTE_ENABLE_MPIIO_ADAPTER "Enable MPI-IO adapter" OFF)
+option(WRP_CTE_ENABLE_STDIO_ADAPTER "Enable STDIO adapter" ON)
+option(WRP_CTE_ENABLE_MPIIO_ADAPTER "Enable MPI-IO adapter" ON)
 option(WRP_CTE_ENABLE_VFD "Enable HDF5 VFD adapter" OFF)
 option(WRP_CTE_ENABLE_NVIDIA_GDS_ADAPTER "Enable NVIDIA GDS adapter" OFF)
-option(WRP_CTE_ENABLE_ADIOS2_ADAPTER "Enable ADIOS2 adapter" OFF)
+option(WRP_CTE_ENABLE_ADIOS2_ADAPTER "Enable ADIOS2 adapter" ON)
 option(WRP_CTE_ENABLE_FUSE_ADAPTER "Enable FUSE3 adapter" OFF)
+option(WRP_CTE_ENABLE_EBPF_ADAPTER "Enable eBPF I/O interceptor adapter" OFF)
 
 #------------------------------------------------------------------------------
 # CAE (context-assimilation-engine) Options
@@ -270,10 +298,21 @@ if(WRP_CORE_STATIC_DEPS)
     endif()
 endif()
 
-# Cereal - find from system or install.sh installation
-find_package(cereal REQUIRED)
+# Cereal - find from system, install.sh installation, or FetchContent
+find_package(cereal CONFIG QUIET)
 if(cereal_FOUND)
     message(STATUS "found cereal at ${cereal_DIR}")
+else()
+    # Fallback to FetchContent if not found on system
+    include(FetchContent)
+    FetchContent_Declare(
+        cereal
+        GIT_REPOSITORY https://github.com/USCiLab/cereal.git
+        GIT_TAG v1.3.2
+        GIT_SHALLOW TRUE
+    )
+    FetchContent_MakeAvailable(cereal)
+    message(STATUS "cereal fetched and configured via FetchContent")
 endif()
 
 # MessagePack C library (pure C, no Boost dependency)
@@ -362,8 +401,12 @@ if(WRP_CTE_ENABLE_COMPRESS)
     pkg_check_modules(libbrotlicommon REQUIRED libbrotlicommon libbrotlidec libbrotlienc)
     message(STATUS "found libbrotli.h at ${libbrotlicommon_INCLUDE_DIRS}")
 
-    pkg_check_modules(blosc2 REQUIRED blosc2)
-    message(STATUS "found blosc2.h at ${blosc2_INCLUDE_DIRS}")
+    pkg_check_modules(blosc2 blosc2)
+    if(blosc2_FOUND)
+        message(STATUS "found blosc2.h at ${blosc2_INCLUDE_DIRS}")
+    else()
+        message(STATUS "blosc2 not found - skipping blosc2 compression support")
+    endif()
 
     # bzip2
     find_path(bzip2_INCLUDE_DIR bzlib.h REQUIRED)
@@ -390,16 +433,16 @@ if(WRP_CTE_ENABLE_COMPRESS)
         message(FATAL_ERROR "lzo not found. Install with: apt-get install liblzo2-dev")
     endif()
 
-    # snappy
-    find_path(snappy_INCLUDE_DIR snappy.h REQUIRED)
-    find_library(snappy_LIBRARY snappy REQUIRED)
+    # snappy (optional)
+    find_path(snappy_INCLUDE_DIR snappy.h)
+    find_library(snappy_LIBRARY snappy)
     if(snappy_INCLUDE_DIR AND snappy_LIBRARY)
         set(snappy_INCLUDE_DIRS ${snappy_INCLUDE_DIR})
         set(snappy_LIBRARIES snappy)
         get_filename_component(snappy_LIBRARY_DIRS "${snappy_LIBRARY}" DIRECTORY)
         message(STATUS "found snappy.h at ${snappy_INCLUDE_DIRS}")
     else()
-        message(FATAL_ERROR "snappy not found. Install with: apt-get install libsnappy-dev")
+        message(STATUS "snappy not found - skipping snappy compression support")
     endif()
 
     # LibPressio - try find_package first (CMake-based), fall back to pkg-config
@@ -590,12 +633,19 @@ find_package(Poco COMPONENTS Net NetSSL Crypto JSON QUIET)
 find_package(nlohmann_json QUIET)
 
 if (WRP_CTE_ENABLE_ADIOS2_ADAPTER OR WRP_CORE_ENABLE_GRAY_SCOTT)
-    # Find ADIOS2
-    find_package(ADIOS2 REQUIRED)
+    # Find ADIOS2 (optional for FUSE builds)
+    find_package(ADIOS2 QUIET)
     if(ADIOS2_FOUND)
         message(STATUS "found ADIOS2 at ${ADIOS2_DIR}")
     else()
-        message(FATAL_ERROR "WRP_CORE_ENABLE_GRAY_SCOTT is ON but ADIOS2 not found. Please install ADIOS2 or disable the option.")
+        if(WRP_CORE_ENABLE_GRAY_SCOTT)
+            message(WARNING "WRP_CORE_ENABLE_GRAY_SCOTT is ON but ADIOS2 not found - disabling Gray-Scott")
+            set(WRP_CORE_ENABLE_GRAY_SCOTT OFF)
+        endif()
+        if(WRP_CTE_ENABLE_ADIOS2_ADAPTER)
+            message(STATUS "ADIOS2 not found - ADIOS2 adapter will be disabled")
+            set(WRP_CTE_ENABLE_ADIOS2_ADAPTER OFF)
+        endif()
     endif()
 endif()
 

README.md

@@ -108,6 +108,20 @@ Heterogeneous-aware, multi-tiered, dynamic I/O buffering system designed to acce
 
 **[Read more →](context-transfer-engine/README.md)**
 
+#### Rust Bindings
+**Location:** [`context-transfer-engine/wrapper/rust/`](context-transfer-engine/wrapper/rust/)
+
+Native Rust bindings for the CTE API, providing idiomatic Rust interfaces with async support.
+
+**Key Features:**
+- Full CTE API coverage with async (default) and sync APIs
+- Tiered storage management with blob scoring
+- Telemetry and monitoring support
+- Thread-safe with proper Rust idioms
+- CXX-based FFI for safe interop
+
+**[Read more →](context-transfer-engine/wrapper/rust/README.md)**
+
 ### 4. Context Assimilation Engine
 **Location:** [`context-assimilation-engine/`](context-assimilation-engine/)
 
@@ -329,6 +343,48 @@ int main() {
 }
 ```
 
+### Context Transfer Engine Rust Example
+
+```rust
+use wrp_cte::{Client, Tag, PoolQuery};
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    // Initialize (starts embedded runtime)
+    let client = Client::new().await?;
+
+    // Create a tag
+    let tag = Tag::new("my_dataset").await?;
+
+    // Store data with automatic scoring
+    tag.put_blob("data.bin".to_string(), vec![1, 2, 3], 0, 0.9).await?;
+
+    // Retrieve data
+    let data = tag.get_blob("data.bin".to_string(), 3, 0).await?;
+    println!("Read {} bytes", data.len());
+
+    // Get telemetry
+    let telemetry = client.poll_telemetry(0).await?;
+    for entry in telemetry {
+        println!("Op {:?}: {} bytes at offset {}", entry.op, entry.size, entry.off);
+    }
+
+    Ok(())
+}
+```
+
+**Build with Cargo:**
+```bash
+cd context-transfer-engine/wrapper/rust
+cargo run --features async
+```
+
+**CMake integration:**
+```cmake
+find_package(iowarp-core REQUIRED)
+# Rust bindings built automatically with -DWRP_CORE_ENABLE_RUST=ON
+```
+
 **Build and Link:**
 ```cmake
 # Unified package includes everything - HermesShm, Chimaera, and all ChiMods
@@ -372,6 +428,14 @@ ctest -R cte                # Context transfer engine tests
 ctest -R omni               # Context assimilation engine tests
 ```
 
+**Rust tests:**
+```bash
+cd context-transfer-engine/wrapper/rust
+cargo test --features async    # Run async API tests
+cargo test --features sync     # Run sync API tests
+cargo test                     # Run all Rust tests
+```
+
 ## Benchmarking
 
 IOWarp Core includes performance benchmarks for measuring runtime and I/O throughput.
@@ -462,6 +526,7 @@ Comprehensive documentation is available for each component:
   - [MODULE_DEVELOPMENT_GUIDE.md](context-transport-primitives/docs/MODULE_DEVELOPMENT_GUIDE.md): Complete ChiMod development guide
 - **[Context Transfer Engine](context-transfer-engine/README.md)**: I/O buffering and acceleration
   - [CTE API Documentation](context-transfer-engine/docs/cte/cte.md): Complete API reference
+  - [Context Transfer Engine Rust Bindings](context-transfer-engine/wrapper/rust/README.md): Rust API reference and examples
 - **[Context Assimilation Engine](context-assimilation-engine/README.md)**: Data ingestion and processing
 - **[Context Exploration Engine](context-exploration-engine/README.md)**: Interactive data exploration
 

chimaera.yaml

@@ -0,0 +1,37 @@
+# IOWarp CTE Storage Configuration
+# Multiple tiered storage devices
+
+devices:
+  # RAM - fastest, smallest
+  - name: ram
+    type: ramfs
+    capacity: 512M
+    path: /dev/shm/chimaera_ram
+
+  # NVMe - fast SSD
+  - name: nvme
+    type: file
+    capacity: 1g
+    path: /mnt/nvme/nrajesh/chimaera_nvme
+
+  # SSD - medium speed
+  - name: ssd
+    type: file
+    capacity: 2g
+    path: /mnt/ssd/nrajesh/chimaera_ssd
+
+  # HDD - slow, largest capacity
+  - name: hdd
+    type: file
+    capacity: 4g
+    path: /mnt/hdd/nrajesh/chimaera_hdd
+
+# Shared memory configuration
+main_segment_size: 1G
+client_data_segment_size: 512M
+runtime_data_segment_size: 512M
+
+# Scheduler configuration
+sched:
+  workers: 4
+  queue_depth: 1024

cmake/AutoConfigureRust.cmake

@@ -0,0 +1,79 @@
+# AutoConfigureRust.cmake
+# Automatically finds and configures Rust toolchain for IOWarp Core
+# This module should be included after WRP_CORE_ENABLE_RUST is set
+
+if(WRP_CORE_ENABLE_RUST AND (NOT Rust_COMPILER OR NOT Rust_CARGO))
+    message(STATUS "Auto-configuring Rust toolchain...")
+
+    # Try to find rustup
+    find_program(RUSTUP_EXE rustup
+        PATHS
+            "$ENV{HOME}/.cargo/bin"
+            "$ENV{CARGO_HOME}/bin"
+            "$ENV{RUSTUP_HOME}/bin"
+        DOC "Rustup executable"
+    )
+
+    if(RUSTUP_EXE)
+        message(STATUS "Found rustup: ${RUSTUP_EXE}")
+
+        # Get the active toolchain
+        execute_process(
+            COMMAND ${RUSTUP_EXE} show active-toolchain
+            OUTPUT_VARIABLE RUSTUP_ACTIVE_TOOLCHAIN
+            OUTPUT_STRIP_TRAILING_WHITESPACE
+            ERROR_QUIET
+        )
+
+        if(RUSTUP_ACTIVE_TOOLCHAIN)
+            # Extract toolchain name (format: "toolchain-name (active)" or just "toolchain-name")
+            string(REGEX MATCH "^[^ ]+" RUSTUP_TOOLCHAIN_NAME "${RUSTUP_ACTIVE_TOOLCHAIN}")
+            message(STATUS "Active rustup toolchain: ${RUSTUP_TOOLCHAIN_NAME}")
+
+            # Get toolchain path
+            execute_process(
+                COMMAND ${RUSTUP_EXE} which rustc
+                OUTPUT_VARIABLE RUSTUP_RUSTC_PATH
+                OUTPUT_STRIP_TRAILING_WHITESPACE
+                ERROR_QUIET
+            )
+
+            if(RUSTUP_RUSTC_PATH)
+                get_filename_component(RUSTUP_TOOLCHAIN_DIR "${RUSTUP_RUSTC_PATH}" DIRECTORY)
+                message(STATUS "Rust toolchain directory: ${RUSTUP_TOOLCHAIN_DIR}")
+
+                if(NOT Rust_COMPILER)
+                    set(Rust_COMPILER "${RUSTUP_TOOLCHAIN_DIR}/rustc" CACHE FILEPATH "Rust compiler" FORCE)
+                    message(STATUS "Auto-set Rust_COMPILER: ${Rust_COMPILER}")
+                endif()
+
+                if(NOT Rust_CARGO)
+                    set(Rust_CARGO "${RUSTUP_TOOLCHAIN_DIR}/cargo" CACHE FILEPATH "Cargo" FORCE)
+                    message(STATUS "Auto-set Rust_CARGO: ${Rust_CARGO}")
+                endif()
+
+                # Also set Rust_RUSTUP to help Corrosion's FindRust
+                set(Rust_RUSTUP "${RUSTUP_EXE}" CACHE FILEPATH "Rustup executable" FORCE)
+                message(STATUS "Auto-set Rust_RUSTUP: ${Rust_RUSTUP}")
+            endif()
+        else()
+            message(WARNING "Could not determine active rustup toolchain")
+        endif()
+    else()
+        message(WARNING "rustup not found - Rust auto-configuration will not work")
+    endif()
+
+    # Auto-find cereal if not already set
+    if(NOT cereal_DIR)
+        # Try spack locations
+        file(GLOB SPACK_CEREAL_DIRS
+            "$ENV{HOME}/spack/opt/spack/*/cereal*/lib*/cmake/cereal"
+            "/opt/spack/opt/spack/*/cereal*/lib*/cmake/cereal"
+        )
+        if(SPACK_CEREAL_DIRS)
+            list(GET SPACK_CEREAL_DIRS 0 SPACK_CEREAL_DIR)
+            set(cereal_DIR "${SPACK_CEREAL_DIR}" CACHE PATH "cereal directory" FORCE)
+            message(STATUS "Auto-found spack cereal: ${cereal_DIR}")
+        endif()
+    endif()
+endif()