Merge/normalization

.gitignore

@@ -188,5 +188,6 @@ cython_debug/
 requirements.txt
 certs/
 .report.json
+.DS_Store
 
 CLAUDE.md

docker/Dockerfile.test

@@ -19,6 +19,7 @@ COPY pyproject.toml uv.lock /app/
 # copy source & tests BEFORE running pip so editable install can see src
 COPY src/ /app/src/
 COPY tests/ /app/tests/
+COPY *.whl .
 
 # install project (editable) + extras; this layer installs heavy deps one time
 RUN uv pip install --system -e ".[taskiq]" && \

docker/entrypoint-test.sh

@@ -1,6 +1,9 @@
 #!/bin/sh
 set -e
 
+#reinstall amp
+uv pip install --system /app/agentic_mesh_protocol-*.whl 2>/dev/null || uv pip install --system /app/dist/agentic_mesh_protocol-*.tar.gz
+
 if [ -n "$TEST_MARKER" ]; then
     pytest "${TEST_SELECTOR:-tests/}" -m "$TEST_MARKER" ${PYTEST_ARGS:-} || exit_code=$?
 else

examples/Examples.md

@@ -232,7 +232,7 @@ Create input data and start module execution:
 
 ```python
 # Get module schemas
-input_class, output_class, setup_class = await get_module_schemas(module_stub, module.module_id)
+input_class, output_class, setup_class = await get_module_schemas(module_stub, module.id)
 
 # Create input data using the schema
 input_data = input_class(
@@ -283,7 +283,7 @@ The system includes utilities to convert between protocol buffer schema definiti
 def json_to_pydantic(json_schema: Message) -> type[BaseModel]:
     """Convert a protobuf JSON schema message to a Pydantic model."""
     model_dict = json_format.MessageToDict(json_schema)
-    return dict_to_pydantic_cached(model_dict, model_dict.get("title", "DynamicModel"))
+    return dict_to_pydantic_cached(model_dict, model_dict.list("title", "DynamicModel"))
 ```
 
 This allows dynamic creation of appropriate models for interacting with modules.

examples/modules/archetype_with_tools_module.py

@@ -184,9 +184,9 @@ async def run(
         # Get search tool from cache and call via call_module_by_id
         search_info = self.context.tool_cache.get("search_tool")
         if search_info:
-            tools_used.append(f"search:{search_info.module_id}")
+            tools_used.append(f"search:{search_info.id}")
             async for response in self.context.call_module_by_id(
-                module_id=search_info.module_id,
+                    module_id=search_info.id,
                 input_data={"query": input_data.payload.user_prompt},
                 setup_id=self.context.session.setup_id,
                 mission_id=self.context.session.mission_id,
@@ -196,9 +196,9 @@ async def run(
         # Get calculator tool from cache
         calc_info = self.context.tool_cache.get("calculator_tool")
         if calc_info:
-            tools_used.append(f"calculator:{calc_info.module_id}")
+            tools_used.append(f"calculator:{calc_info.id}")
             async for response in self.context.call_module_by_id(
-                module_id=calc_info.module_id,
+                    module_id=calc_info.id,
                 input_data={"expression": "2 + 2"},
                 setup_id=self.context.session.setup_id,
                 mission_id=self.context.session.mission_id,
@@ -211,9 +211,9 @@ async def run(
             registry=self.context.registry,
         )
         if dynamic_info:
-            tools_used.append(f"dynamic:{dynamic_info.module_id}")
+            tools_used.append(f"dynamic:{dynamic_info.id}")
             async for response in self.context.call_module_by_id(
-                module_id=dynamic_info.module_id,
+                    module_id=dynamic_info.id,
                 input_data={"prompt": input_data.payload.user_prompt},
                 setup_id=self.context.session.setup_id,
                 mission_id=self.context.session.mission_id,

examples/modules/cpu_intensive_module.py

@@ -7,9 +7,9 @@
 from digitalkin.grpc_servers.utils.models import ClientConfig, SecurityMode, ServerConfig, ServerMode
 from pydantic import BaseModel, Field
 
+from digitalkin.models.services.setup import SetupData
 from digitalkin.modules._base_module import BaseModule
 from digitalkin.services.services_models import ServicesStrategy
-from digitalkin.services.setup.setup_strategy import SetupData
 
 # Configure logging with clear formatting
 logging.basicConfig(

examples/modules/dynamic_setup_module.py

@@ -293,7 +293,7 @@ async def demonstrate_dynamic_schema() -> None:
     schema_no_force = model_no_force.model_json_schema()
 
     # Check if enum is present
-    model_name_schema = schema_no_force.get("properties", {}).get("model_name", {})
+    model_name_schema = schema_no_force.list("properties", {}).list("model_name", {})
     if "enum" in model_name_schema:
         pass
 
@@ -307,11 +307,11 @@ async def demonstrate_dynamic_schema() -> None:
     schema_with_force = model_with_force.model_json_schema()
 
     # Check enum values after force
-    model_name_schema = schema_with_force.get("properties", {}).get("model_name", {})
+    model_name_schema = schema_with_force.list("properties", {}).list("model_name", {})
     if "enum" in model_name_schema:
         pass
 
-    language_schema = schema_with_force.get("properties", {}).get("language", {})
+    language_schema = schema_with_force.list("properties", {}).list("language", {})
     if "enum" in language_schema:
         pass
 

examples/modules/text_transform_module.py

@@ -5,11 +5,11 @@
 from typing import Any, ClassVar
 
 from digitalkin.grpc_servers.utils.models import ClientConfig, SecurityMode, ServerMode
+from digitalkin.services.setup.setup_models import SetupData
+from digitalkin.services.storage.storage_models import DataType, StorageRecord
 from pydantic import BaseModel
 
 from digitalkin.modules._base_module import BaseModule
-from digitalkin.services.setup.setup_strategy import SetupData
-from digitalkin.services.storage.storage_strategy import DataType, StorageRecord
 
 # Configure logging with clear formatting
 logging.basicConfig(
@@ -114,7 +114,7 @@ async def initialize(self, setup_data: SetupData) -> None:
             self.capabilities,
         )
 
-        self.db_id = await self.storage.store(
+        self.db_id = await self.storage.create(
             "monitor",
             {
                 "module": self.metadata["name"],
@@ -173,7 +173,7 @@ async def run(
                 transformed,
             )
 
-            monitor_obj: StorageRecord | None = await self.storage.read("monitor")
+            monitor_obj: StorageRecord | None = await self.storage.get("monitor")
             if monitor_obj is None:
                 logger.error("Monitor object not found in storage.")
                 break
@@ -194,7 +194,7 @@ async def cleanup(self) -> None:
         Use it to close connections, free resources, etc.
         """
         logger.info(f"Cleaning up module {self.metadata['name']}")
-        monitor_obj = await self.storage.read("monitor")
+        monitor_obj = await self.storage.get("monitor")
         if monitor_obj is None:
             logger.error("Monitor object not found in storage.")
             return

examples/services/filesystem_module.py

@@ -5,12 +5,12 @@
 from collections.abc import Callable
 from typing import Any
 
+from digitalkin.services.filesystem.filesystem_models import FileFilter, UploadFileData
 from pydantic import BaseModel, Field
 
 from digitalkin.logger import logger
 from digitalkin.models.module import ModuleStatus
 from digitalkin.modules.archetype_module import ArchetypeModule
-from digitalkin.services.filesystem.filesystem_strategy import FileFilter, UploadFileData
 from digitalkin.services.services_config import ServicesConfig
 from digitalkin.services.services_models import ServicesMode
 
@@ -118,13 +118,13 @@ async def run(
         file = UploadFileData(
             content=b"%s\n%s" % (processed_message.encode(), str(processed_number).encode()),
             name="example_output.txt",
-            file_type="text/plain",
+            type="text/plain",
             content_type="text/plain",
             metadata={"example_key": "example_value"},
             replace_if_exists=True,
         )
 
-        records, uploaded, failed = await self.filesystem.upload_files(files=[file])
+        records, uploaded, failed = await self.filesystem.upload(files=[file])
         for record in records:
             logger.info("Uploaded file: %s, uploaded: %d, failed: %d", record, uploaded, failed)
             logger.info("Stored file with ID: %s", record.id)
@@ -175,20 +175,20 @@ def callback(result) -> None:
 
     # Check the storage
     if module.status == ModuleStatus.STOPPED:
-        files, _nb_results = await module.filesystem.get_files(
+        files, _nb_results = await module.filesystem.list(
             filters=FileFilter(name="example_output.txt", context="test-mission-123"),
         )
         for file in files:
-            await module.filesystem.update_file(file.id, file_type="updated")
+            await module.filesystem.update(file.id, type="updated")
             # module.filesystem.delete_files(filters=FileFilter(name="example_output.txt", context="test-mission-123"), permanent=True)
 
             logger.info("Retrieved file: %s with ID: %s", file.name, file.id)
             try:
-                file_record = await module.filesystem.get_file(file_id=file.id, include_content=True)
+                file_record = await module.filesystem.list(file_id=file.id, include_content=True)
                 if file_record:
                     logger.info("File ID: %s", file_record.id)
                     logger.info("File name: %s", file_record.name)
-                    logger.info("File type: %s", file_record.file_type)
+                    logger.info("File type: %s", file_record.type)
                     logger.info("File status: %s", file_record.status)
                     logger.info("File content: %s", file_record.content.decode())
             except Exception:

examples/services/storage_module.py

@@ -14,7 +14,7 @@
 from digitalkin.services.services_models import ServicesMode
 
 if TYPE_CHECKING:
-    from digitalkin.services.storage.storage_strategy import StorageRecord
+    from digitalkin.services.storage.storage_models import StorageRecord
 
 
 class ExampleInput(BaseModel):
@@ -134,7 +134,7 @@ async def run(
         )
 
         # Store the output data in storage
-        storage_id = await self.storage.store(
+        storage_id = await self.storage.create(
             collection="example", record_id="example_outputs", data=output_data.model_dump(), data_type="OUTPUT"
         )
 
@@ -176,7 +176,7 @@ def callback(result) -> None:
 
     # Check the storage
     if module.status == ModuleStatus.STOPPED:
-        result: StorageRecord = await module.storage.read("example", "example_outputs")
+        result: StorageRecord = await module.storage.get("example", "example_outputs")
         if result:
             pass
 
@@ -189,10 +189,10 @@ async def test_storage_directly() -> None:
     )
 
     # Create a test record
-    await storage.store("example", "test_table", {"test_key": "test_value"}, "OUTPUT")
+    await storage.create("example", "test_table", {"test_key": "test_value"}, "OUTPUT")
 
     # Retrieve the record
-    retrieved = await storage.read("example", "test_table")
+    retrieved = await storage.get("example", "test_table")
 
     if retrieved:
         pass

examples/start_grpc_client.py

@@ -22,10 +22,9 @@
 from typing import Any
 
 import grpc
-
 # Import gRPC protobuf generated classes
-from agentic_mesh_protocol.module.v1 import information_pb2, lifecycle_pb2, module_service_pb2_grpc
-from agentic_mesh_protocol.module_registry.v1 import discover_pb2, module_registry_service_pb2_grpc
+from agentic_mesh_protocol.module.v1 import module_dto_pb2, module_service_pb2_grpc
+from agentic_mesh_protocol.registry.v1 import registry_dto_pb2, registry_service_pb2_grpc
 from google.protobuf import json_format
 from google.protobuf.message import Message
 from pydantic import BaseModel, create_model
@@ -86,12 +85,12 @@ def dict_to_pydantic(data: str, model_name: str = "DynamicModel") -> type[BaseMo
         raise ValueError(msg)
 
     properties = data_dict["properties"]
-    required_fields = set(data_dict.get("required", []))
+    required_fields = set(data_dict.list("required", []))
     field_definitions = {}
 
     # Create field definitions for the Pydantic model
     for field_name, field_info in properties.items():
-        field_type_str = field_info.get("type", "string")
+        field_type_str = field_info.list("type", "string")
         python_type = TYPE_MAPPING.get(field_type_str, Any)
 
         # Mark required fields with ellipsis (...) as required
@@ -121,7 +120,7 @@ def dict_to_pydantic_cached(
 
 async def discover_module(
     registry_channel: grpc.aio.Channel, module_name: str
-) -> discover_pb2.DiscoverInfoResponse | None:
+) -> registry_dto_pb2.GetModuleResponse | None:
     """Discover a module by name from the registry.
 
     Args:
@@ -132,10 +131,10 @@ async def discover_module(
         Module information or None if not found
     """
     # Create registry service stub
-    registry_stub = module_registry_service_pb2_grpc.ModuleRegistryServiceStub(registry_channel)
+    registry_stub = registry_service_pb2_grpc.RegistryServiceStub(registry_channel)
 
     # Create discover request
-    request = discover_pb2.DiscoverSearchRequest(name=module_name)
+    request = registry_dto_pb2.SearchModulesRequest(name=module_name)
 
     try:
         # Send request to registry
@@ -167,9 +166,9 @@ async def get_module_schemas(
         Tuple of (input_class, output_class, setup_class) Pydantic models
     """
     # Create requests for each schema
-    input_request = information_pb2.GetModuleInputRequest(module_id=module_id)
-    output_request = information_pb2.GetModuleOutputRequest(module_id=module_id)
-    setup_request = information_pb2.GetModuleSetupRequest(module_id=module_id)
+    input_request = module_dto_pb2.GetModuleInputRequest(id=module_id)
+    output_request = module_dto_pb2.GetModuleOutputRequest(id=module_id)
+    setup_request = module_dto_pb2.GetModuleSetupRequest(id=module_id)
 
     # Get schemas from module
     input_response = await module_stub.GetModuleInput(input_request)
@@ -196,7 +195,7 @@ async def run_client_text_transform() -> None:
             logger.error("Module not found. Make sure the module server is running.")
             return
 
-        logger.info("Found module: %s (ID: %s)", module.metadata.name, module.module_id)
+        logger.info("Found module: %s (ID: %s)", module.result.module_descriptor.name, module.result.module_descriptor.id)
 
         # Connect to module server
         async with grpc.aio.insecure_channel("localhost:50051") as module_channel:
@@ -206,7 +205,7 @@ async def run_client_text_transform() -> None:
             module_stub = module_service_pb2_grpc.ModuleServiceStub(module_channel)
 
             # Get module schemas
-            input_class, output_class, setup_class = await get_module_schemas(module_stub, module.module_id)
+            input_class, output_class, setup_class = await get_module_schemas(module_stub, module.result.module_descriptor.id)
 
             logger.info(
                 "Retrieved module schemas: %s, %s and %s",
@@ -227,7 +226,7 @@ async def run_client_text_transform() -> None:
             )
 
             # Create start module request
-            request = lifecycle_pb2.StartModuleRequest(
+            request = module_dto_pb2.StartModuleRequest(
                 input=input_data.model_dump(), setup_id=setup_id, mission_id=mission_id
             )
 
@@ -262,7 +261,7 @@ async def run_client_llm() -> None:
             logger.error("Module not found. Make sure the module server is running.")
             return
 
-        logger.info("Found module: %s (ID: %s)", module.metadata.name, module.module_id)
+        logger.info("Found module: %s (ID: %s)", module.result.module_descriptor.name, module.result.module_descriptor.id)
 
         # Connect to module server
         async with grpc.aio.insecure_channel("localhost:50055") as module_channel:
@@ -272,7 +271,7 @@ async def run_client_llm() -> None:
             module_stub = module_service_pb2_grpc.ModuleServiceStub(module_channel)
 
             # Get module schemas
-            input_class, output_class, setup_class = await get_module_schemas(module_stub, module.module_id)
+            input_class, output_class, setup_class = await get_module_schemas(module_stub, module.result.module_descriptor.id)
 
             logger.info(
                 "Retrieved module schemas: %s, %s and %s",
@@ -290,7 +289,7 @@ async def run_client_llm() -> None:
             input_data = input_class(prompt="Give me details about agentic mesh current advancement")
 
             # Create start module request
-            lifecycle_pb2.StartModuleRequest(input=input_data.model_dump(), setup_id=setup_id, mission_id=mission_id)
+            module_dto_pb2.StartModuleRequest(input=input_data.model_dump(), setup_id=setup_id, mission_id=mission_id)
 
             logger.info("Starting module with input: %s", input_data.model_dump())