[DAPS-1912] - refactor: register linkml storage engine with schema handler

core/server/ClientWorker.cpp

@@ -841,7 +841,7 @@ ClientWorker::procRecordCreateRequest(const std::string &a_uid,
 
   if (request->has_metadata() && request->has_sch_id()) {
     validator_err = m_schema_handler->validateMetadataContent(
-        request->sch_id(), request->metadata(), log_context);
+        a_uid, request->sch_id(), request->metadata(), log_context);
 
     if (!validator_err.empty()) {
       DL_ERROR(log_context, "Metadata validation error: " << validator_err);
@@ -942,7 +942,7 @@ ClientWorker::procRecordUpdateRequest(const std::string &a_uid,
 
       if (validator_err.empty()) {
         validator_err = m_schema_handler->validateMetadataContent(
-            sch_id, effective_metadata, log_context);
+            a_uid, sch_id, effective_metadata, log_context);
 
         if (!validator_err.empty()) {
           DL_WARNING(log_context,

core/server/Config.hpp

@@ -4,6 +4,7 @@
 
 // Core local private includes
 #include "AuthenticationManager.hpp"
+#include "SchemaAPIConfig.hpp"
 
 // DataFed Common public includes
 #include "common/DynaLog.hpp"
@@ -16,6 +17,7 @@
 #include <mutex>
 #include <stdint.h>
 #include <string>
+#include <unordered_map>
 
 namespace SDMS {
 namespace Core {
@@ -79,6 +81,8 @@ class Config {
   uint32_t metrics_purge_period;
   uint32_t metrics_purge_age;
 
+  std::unordered_map<std::string, SchemaAPIConfig> schemas;
+
   // MsgComm::SecurityContext            sec_ctx;
   std::unique_ptr<ICredentials> sec_ctx;
 

core/server/DatabaseAPI.cpp

@@ -2955,6 +2955,12 @@ void DatabaseAPI::schemaCreate(const SDMS::SchemaCreateRequest &a_request,
   payload["pub"] = a_request.pub();
   payload["sys"] = a_request.sys();
   payload["def"] = a_request.def();
+  if (!a_request.format().empty()) {
+    payload["format"] = a_request.format();
+  }
+  if (!a_request.type().empty()) {
+    payload["type"] = a_request.type();
+  }
   string body = payload.dump(-1, ' ', true);
 
   dbPost("schema/create", {}, &body, result, log_context);
@@ -3058,29 +3064,45 @@ void DatabaseAPI::setSchemaData(SchemaData *a_schema,
   a_schema->set_id(a_obj.getString("id"));
   a_schema->set_ver(a_obj.getNumber("ver"));
 
-  if (a_obj.has("cnt"))
+  if (a_obj.has("cnt")) {
     a_schema->set_cnt(a_obj.asNumber());
+  }
 
-  if (a_obj.has("own_id") && !a_obj.value().isNull())
+  if (a_obj.has("own_id") && !a_obj.value().isNull()) {
     a_schema->set_own_id(a_obj.asString());
+  }
 
-  if (a_obj.has("own_nm") && !a_obj.value().isNull())
+  if (a_obj.has("own_nm") && !a_obj.value().isNull()) {
     a_schema->set_own_nm(a_obj.asString());
+  }
 
-  if (a_obj.has("desc"))
+  if (a_obj.has("desc")) {
     a_schema->set_desc(a_obj.asString());
+  }
 
-  if (a_obj.has("pub"))
+  if (a_obj.has("pub")) {
     a_schema->set_pub(a_obj.asBool());
+  }
 
-  if (a_obj.has("depr"))
+  if (a_obj.has("depr")) {
     a_schema->set_depr(a_obj.asBool());
+  }
 
-  if (a_obj.has("ref"))
+  if (a_obj.has("ref")) {
     a_schema->set_ref(a_obj.asBool());
+  }
 
-  if (a_obj.has("def"))
+  if (a_obj.has("def")) {
     a_schema->set_def(a_obj.value().toString());
+  }
+
+  if (a_obj.has("type")) {
+    a_schema->set_type(a_obj.asString());
+  }
+
+  if (a_obj.has("format")) {
+    a_schema->set_format(a_obj.asString());
+  }
 
   Value::ArrayConstIter j;
   SchemaData *dep;

core/server/client_handlers/SchemaHandler.cpp

@@ -3,7 +3,10 @@
 #include "LocalJsonErrorHandler.hpp"
 #include "common/TraceException.hpp"
 #include "schema_storage/ArangoSchemaStorage.hpp"
+#include "schema_storage/ExternalSchemaStorage.hpp"
 #include "schema_validators/JsonSchemaValidator.hpp"
+#include "schema_validators/ExternalSchemaValidator.hpp"
+#include "Config.hpp"
 
 // Standard includes
 #include <functional>
@@ -13,8 +16,19 @@ using namespace std;
 namespace SDMS {
 namespace Core {
 
-SchemaHandler::SchemaHandler(DatabaseAPI &a_db_client)
+  SchemaHandler::SchemaHandler(DatabaseAPI &a_db_client)
     : m_db_client(a_db_client) {
+
+    if ( Config::getInstance().schemas.count("linkml") ){
+      SchemaAPIConfig linkml_config = Config::getInstance().schemas.at("linkml");
+      auto schema_api_client_storage = make_unique<SchemaAPIClient>(linkml_config);
+      auto linkml_storage = std::make_shared<ExternalSchemaStorage>(std::move(schema_api_client_storage));
+      m_schema_factory.registerStorage("linkml", std::move(linkml_storage));
+      auto schema_api_client_validator = make_unique<SchemaAPIClient>(linkml_config);
+      auto linkml_schema_validator = std::make_shared<ExternalSchemaValidator>(std::move(schema_api_client_validator), "linkml");
+      m_schema_factory.registerValidator("linkml",
+          std::move(linkml_schema_validator));
+    }
     // Assumes that we have already placed the schema in the database, arango
     // storage is a shell to be consistent with the interface.
     auto arango_storage = std::make_shared<ArangoSchemaStorage>();
@@ -32,7 +46,7 @@ SchemaHandler::SchemaHandler(DatabaseAPI &a_db_client)
     m_schema_factory.registerValidator("json-schema",
                                        std::move(json_schema_validator));
     m_schema_factory.setDefaultSchemaType("json-schema");
-}
+  }
 
 // ── Schema Definition Handlers ──────────────────────────────────────────────
 
@@ -62,7 +76,13 @@ void SchemaHandler::handleCreate(const std::string &a_uid,
   }
 
   // Persist to Arango — exception propagates naturally on failure
-  m_db_client.schemaCreate(a_request, a_reply, log_context);
+  // For external schema types, store a stub in Arango —
+  // the real content lives in external storage.
+  SchemaCreateRequest arango_req(a_request);
+  if (!a_request.type().empty() && a_request.type() != "json-schema") {
+    arango_req.set_def("{}");
+  }
+  m_db_client.schemaCreate(arango_req, a_reply, log_context);
 
   // Store content through factory (no-op for Arango, meaningful for external)
   try {
@@ -140,7 +160,13 @@ void SchemaHandler::handleRevise(const std::string &a_uid,
   }
 
   // Create new revision in Arango
-  m_db_client.schemaRevise(a_request, a_reply, log_context);
+  // For external schema types, store a stub in Arango —
+  // the real content lives in external storage.
+  SchemaReviseRequest arango_req(a_request);
+  if (!schema_type.empty() && schema_type != "json-schema") {
+    arango_req.set_def("{}");
+  }
+  m_db_client.schemaRevise(arango_req, a_reply, log_context);
 
   // Store content for new revision only when def was provided
   if (a_request.has_def()) {
@@ -220,7 +246,13 @@ void SchemaHandler::handleUpdate(const std::string &a_uid,
   }
 
   // Update in-place in Arango
-  m_db_client.schemaUpdate(a_request, log_context);
+  // For external schema types, store a stub in Arango —
+  // the real content lives in external storage.
+  SchemaUpdateRequest arango_req(a_request);
+  if (!schema_type.empty() && schema_type != "json-schema") {
+    arango_req.set_def("{}");
+  }
+  m_db_client.schemaUpdate(arango_req, log_context);
 
   // Update content in external storage only when def was provided.
   // Uses updateContent (in-place) rather than storeContent (new entry).
@@ -250,7 +282,9 @@ void SchemaHandler::handleUpdate(const std::string &a_uid,
       rollback_request.set_id(sch_doc.getString("id"));
       rollback_request.set_desc(sch_doc.getString("desc"));
       rollback_request.set_pub(sch_doc.getBool("pub"));
-      rollback_request.set_sys(sch_doc.getBool("sys"));
+      if (sch_doc.has("sys")) {
+        rollback_request.set_sys(sch_doc.getBool("sys"));
+      }
 
       DL_ERROR(log_context,
                "Schema storage update failed attempting rollback: " << e.what());
@@ -269,10 +303,12 @@ void SchemaHandler::handleUpdate(const std::string &a_uid,
 // ── Metadata Validation ─────────────────────────────────────────────────────
 
 std::string SchemaHandler::validateMetadataContent(
+    const std::string &a_uid,
     const std::string &a_sch_id,
     const std::string &a_metadata,
     LogContext log_context) {
 
+  m_db_client.setClient(a_uid);
   DL_DEBUG(log_context, "validateMetadataContent schema=" << a_sch_id);
 
   std::string schema_type = "json-schema";
@@ -342,10 +378,9 @@ void SchemaHandler::handleMetadataValidate(
     LogContext log_context) {
 
   DL_DEBUG(log_context, "Metadata validate");
-  m_db_client.setClient(a_uid);
 
   std::string errors = validateMetadataContent(
-      a_request.sch_id(), a_request.metadata(), log_context);
+      a_uid, a_request.sch_id(), a_request.metadata(), log_context);
 
   if (!errors.empty()) {
     a_reply.set_errors(errors);

core/server/client_handlers/SchemaHandler.hpp

@@ -118,9 +118,10 @@ class SchemaHandler {
                     AckReply &a_reply,
                     LogContext log_context);
 
-  std::string validateMetadataContent(const std::string &a_sch_id,
-                                    const std::string &a_metadata,
-                                    LogContext log_context);
+  std::string validateMetadataContent(const std::string &a_uid,
+                                      const std::string &a_sch_id,
+                                      const std::string &a_metadata,
+                                      LogContext log_context);
 
 private:
   /**

core/server/schema_storage/ExternalSchemaStorage.hpp

@@ -23,7 +23,6 @@ class ExternalSchemaStorage : public ISchemaStorage {
    */
   explicit ExternalSchemaStorage(std::unique_ptr<SchemaAPIClient> a_client);
   ~ExternalSchemaStorage() override = default;
-  // Non-copyable (owns unique_ptr)
   ExternalSchemaStorage(const ExternalSchemaStorage &) = delete;
   ExternalSchemaStorage &operator=(const ExternalSchemaStorage &) = delete;
   std::string storeContent(const std::string &a_id,

core/server/schema_validators/ExternalSchemaValidator.hpp

@@ -44,6 +44,11 @@ class ExternalSchemaValidator : public ISchemaValidator {
 
   bool hasValidationCapability() const override { return true; }
 
+  bool cacheSchema(const std::string &, const std::string &,
+                 const std::string &, LogContext) override {
+    return true;
+  }
+
 private:
   std::unique_ptr<SchemaAPIClient> m_client;
   std::string m_engine;

core/server/tests/integration/CMakeLists.txt

@@ -6,12 +6,12 @@
 
 set(MOCK_SCHEMA_IMAGE "savannah.ornl.gov/datafed/mock-schema:latest"
     CACHE STRING "Docker image for the mock Schema API (Prism)")
-set(MOCK_SCHEMA_PORT "4010"
+set(MOCK_SCHEMA_PORT "4011"
     CACHE STRING "Host port for the mock Schema API container")
 set(MOCK_SCHEMA_PULL "true"
     CACHE STRING "Pull the mock image before starting (set to 'false' for offline/local builds)")
 
-set(ARANGO_IMAGE "savannah.ornl.gov/datafed/arango-foxx:latest"
+set(ARANGO_IMAGE "arangodb/arangodb:3.12.4"
     CACHE STRING "Docker image for the ArangoDB test instance (with Foxx services)")
 set(ARANGO_PORT "8529"
     CACHE STRING "Host port for the ArangoDB test container")
@@ -22,6 +22,12 @@ set(ARANGO_PULL "true"
 set(DATAFED_TEST_DATABASE_NAME "sdms_test"
     CACHE STRING "Database name for integration tests (default: sdms_test)")
 
+# NOTE - Mock I don't think has the v1 in the path, the actual service however does.
+set(DATAFED_TEST_SCHEMA_API_URL "http://0.0.0.0:8080/v1"
+    CACHE STRING "Url to LinkML Schema")
+set(DATAFED_TEST_SCHEMA_API_TOKEN "testy"
+    CACHE STRING "Token for LinkML Schema Service")
+
 # ── Check for Docker (needed by SchemaAPIClient tests) ───────────────────────
 
 find_program(DOCKER_EXECUTABLE docker)
@@ -34,7 +40,8 @@ endif()
 foreach(PROG
     test_SchemaServiceFactory
     test_SchemaAPIClient
-    test_SchemaHandler
+    test_SchemaHandlerJson
+    test_SchemaHandlerLinkML
 )
   file(GLOB ${PROG}_SOURCES ${PROG}*.cpp)
   add_executable(integration_${PROG} ${${PROG}_SOURCES})
@@ -138,6 +145,14 @@ if(DOCKER_EXECUTABLE)
     "DATAFED_TEST_ARANGO_PASS=${ARANGO_ROOT_PASS}"
     "DATAFED_DATABASE_NAME=${ARANGO_TEST_DATABASE_NAME}"
   )
+  set(_linkml_test_env
+    "DATAFED_TEST_ARANGO_URL=http://localhost:${ARANGO_PORT}/_db/${DATAFED_TEST_DATABASE_NAME}/api/${DATAFED_FOXX_MAJOR}/"
+    "DATAFED_TEST_ARANGO_USER=root"
+    "DATAFED_TEST_ARANGO_PASS=${ARANGO_ROOT_PASS}"
+    "DATAFED_DATABASE_NAME=${ARANGO_TEST_DATABASE_NAME}"
+    "DATAFED_TEST_SCHEMA_API_URL=${DATAFED_TEST_SCHEMA_API_URL}"
+    "DATAFED_TEST_SCHEMA_API_TOKEN=${DATAFED_TEST_SCHEMA_API_TOKEN}"
+  )
 
   # ── Fixture setup: start ArangoDB container ──────────────────────────
 
@@ -165,18 +180,28 @@ if(DOCKER_EXECUTABLE)
 
   # ── Attach fixture to SchemaHandler tests ────────────────────────────
 
-  set_tests_properties(integration_test_SchemaHandler PROPERTIES
+  set_tests_properties(integration_test_SchemaHandlerJson PROPERTIES
     FIXTURES_REQUIRED TestArangoDB
     ENVIRONMENT       "${_arango_test_env}"
     LABELS            "integration"
   )
+  set_tests_properties(integration_test_SchemaHandlerLinkML PROPERTIES
+    FIXTURES_REQUIRED TestArangoDB
+    ENVIRONMENT       "${_linkml_test_env}"
+    LABELS            "integration"
+  )
 
   if(ENABLE_MEMORY_TESTS)
-    set_tests_properties(memory_test_SchemaHandler PROPERTIES
+    set_tests_properties(memory_test_SchemaHandlerJson PROPERTIES
       FIXTURES_REQUIRED TestArangoDB
       ENVIRONMENT       "${_arango_test_env}"
       LABELS            "integration;memory"
     )
+    set_tests_properties(memory_test_SchemaHandlerLinkML PROPERTIES
+      FIXTURES_REQUIRED TestArangoDB
+      ENVIRONMENT       "${_linkml_test_env}"
+      LABELS            "integration;memory"
+    )
   endif()
 
 endif()

core/server/tests/integration/start_mock_schema.sh

@@ -6,14 +6,14 @@
 #
 # Optional env:
 #   MOCK_SCHEMA_IMAGE   — Docker image (default: savannah.ornl.gov/datafed/mock-schema:latest)
-#   MOCK_SCHEMA_PORT    — Host port to bind (default: 4010)
+#   MOCK_SCHEMA_PORT    — Host port to bind (default: 4011)
 #   CONTAINER_NAME      — Docker container name (default: datafed-mock-schema)
 #   MOCK_SCHEMA_PULL    — "true" to pull image before starting (default: true)
 
 set -eu
 
 IMAGE="${MOCK_SCHEMA_IMAGE:-savannah.ornl.gov/datafed/mock-schema:latest}"
-PORT="${MOCK_SCHEMA_PORT:-4010}"
+PORT="${MOCK_SCHEMA_PORT:-4011}"
 NAME="${CONTAINER_NAME:-datafed-mock-schema}"
 PULL="${MOCK_SCHEMA_PULL:-true}"
 MAX_WAIT=30
@@ -40,20 +40,30 @@ fi
 
 echo "Starting mock schema server on port ${PORT}..."
 
+echo "docker run -d --name ${NAME} -p "${PORT}:4011" -e PRISM_DYNAMIC=false -e PRISM_ERRORS=true -e PRISM_PORT=4011 ${IMAGE}"
+
 docker run -d \
   --name "${NAME}" \
-  -p "${PORT}:4010" \
+  -p "${PORT}:4011" \
   -e PRISM_DYNAMIC=false \
   -e PRISM_ERRORS=true \
+  -e PRISM_PORT=4011 \
   "${IMAGE}"
 
 # ── Wait for readiness ───────────────────────────────────────────────────────
 # Hit the list endpoint — any 2xx means Prism is serving the spec.
 
 echo "Waiting for readiness (max ${MAX_WAIT}s)..."
 
+if ! curl --version &>/dev/null; then
+  echo "ERROR: curl is broken (likely missing shared library). Check: curl --version" >&2
+  curl --version # Print the actual error for diagnostics
+  exit 1
+fi
+
+echo "Testing: http://localhost:${PORT}/schemas?page=1&limit=10&schema_format=json"
 for i in $(seq 1 ${MAX_WAIT}); do
-  if curl -sf -o /dev/null "http://localhost:${PORT}/schemas?page=1&limit=1" 2>/dev/null; then
+  if curl -sf -o /dev/null "http://localhost:${PORT}/schemas?page=1&limit=10&schema_format=json" 2>/dev/null; then
     echo "Mock schema server ready on port ${PORT} (took ${i}s)"
     exit 0
   fi