feat(eval): coordinator scaling evaluation with race-safe live dispatch

.gitignore

@@ -561,6 +561,9 @@ compile_commands.json
 
 # PRD drafts (local planning)
 PRD_*.md
+
+# setuptools-scm generated version file
+morphling/_version.py
 *.bak
 *.swp
 

Dockerfile

@@ -3,7 +3,7 @@
 
 RUN sed -i 's|http://security.ubuntu.com/ubuntu|http://archive.ubuntu.com/ubuntu|g' /etc/apt/sources.list || true
 
 RUN apt-get update && apt-get upgrade -y
 
 RUN apt-get install -y \
     curl \
@@ -13,7 +13,7 @@
     pkg-config
 
 # Install Nsight Systems (nsys) and Nsight Compute (ncu)
 RUN apt-get update && apt-get install -y \
     nsight-systems-2025.5.2 \
     nsight-compute-2025.4.1
 
@@ -22,20 +22,20 @@
     ninja-build \
     ccache
 
 RUN apt-get install -y \
     libssl-dev \
     protobuf-compiler \
     libgrpc-dev \
     libgrpc++-dev
 
 RUN apt-get install -y \
     libfmt-dev \
     libspdlog-dev
 
 RUN apt-get install -y \
     libevent-dev
 
 RUN apt-get update && apt-get install -y \
     libxml2-dev \
     xsltproc \
     uuid-dev \
@@ -43,7 +43,7 @@
     rapidjson-dev \
     libxslt1-dev
 
 RUN apt-get update && apt-get install -y \
     gdb \
     vim \
     nano \
@@ -57,14 +57,14 @@
 
 # PyTorch 2.11 image uses PEP 668 EXTERNALLY-MANAGED.
 # Safe to remove in a Docker container.
 RUN rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED
 
 # 创建工作目录
 WORKDIR /app
 
 # copy requirements.txt first to leverage Docker cache
 COPY requirements.txt /app/
 RUN pip install --no-cache -r /app/requirements.txt
 
 # Copy the rest of the project files
 COPY . /app/
@@ -77,6 +77,13 @@
 ENV CCACHE_DIR=/ccache
 RUN if [ "$USE_CCACHE" = "1" ]; then ccache -M 5G; fi
 
+# --no-build-isolation (below) does not install build-system.requires, and the
+# build context excludes .git; install setuptools-scm and inject the git-derived
+# version via a build arg (empty => pyproject [tool.setuptools_scm] fallback).
+RUN pip install --no-cache "setuptools-scm>=8"
+ARG MORPHLING_VERSION
+ENV SETUPTOOLS_SCM_PRETEND_VERSION_FOR_MORPHLING=${MORPHLING_VERSION}
+
 # 构建和安装项目（使用系统 python）with BuildKit cache mount for ccache
 RUN --mount=type=cache,target=/ccache \
     if [ "$USE_CCACHE" = "1" ]; then export PATH="/usr/lib/ccache:$PATH"; fi && \

csrc/backend/operation_id.h

@@ -0,0 +1,32 @@
+#pragma once
+
+#include <atomic>
+#include <stdexcept>
+#include <string>
+
+namespace morphling {
+namespace backend {
+
+inline constexpr int kMaxLifetimeOperationCount = 65'536;
+
+inline void ValidateOperationId(int oid) {
+  if (oid < 0 || oid >= kMaxLifetimeOperationCount) {
+    throw std::out_of_range("matmul operation id " + std::to_string(oid) +
+                            " is outside the valid range [0, " +
+                            std::to_string(kMaxLifetimeOperationCount) + ")");
+  }
+}
+
+inline int ReserveOperationId(std::atomic_int& next_oid) noexcept {
+  int oid = next_oid.load(std::memory_order_relaxed);
+  while (oid >= 0 && oid < kMaxLifetimeOperationCount) {
+    if (next_oid.compare_exchange_weak(oid, oid + 1,
+                                       std::memory_order_relaxed)) {
+      return oid;
+    }
+  }
+  return -1;
+}
+
+}  // namespace backend
+}  // namespace morphling

csrc/backend/partition_tracker.cpp

@@ -106,6 +106,47 @@ void PartitionTracker::RemovePartitionByKey(const std::string& partition_key) {
   }
 }
 
+bool PartitionTracker::ReassignPartitionToDevice(
+    const std::string& partition_key, int64_t target_device_id) {
+  std::lock_guard<std::mutex> lock(mutex_);
+
+  auto partition_it = partition_map_.find(partition_key);
+  if (partition_it == partition_map_.end()) {
+    return false;
+  }
+  const auto& partition_info = partition_it->second;
+
+  auto owner_it = partition_to_device_.find(partition_key);
+  if (owner_it != partition_to_device_.end()) {
+    auto device_it = device_partitions_.find(owner_it->second);
+    if (device_it != device_partitions_.end()) {
+      auto& partitions = device_it->second;
+      partitions.erase(
+          std::remove_if(partitions.begin(), partitions.end(),
+                         [&partition_key](const PartitionInfoPtr& candidate) {
+                           return candidate->key == partition_key;
+                         }),
+          partitions.end());
+      if (partitions.empty()) {
+        device_partitions_.erase(device_it);
+      }
+    }
+  }
+
+  auto& target_partitions = device_partitions_[target_device_id];
+  target_partitions.erase(
+      std::remove_if(target_partitions.begin(), target_partitions.end(),
+                     [&partition_key](const PartitionInfoPtr& candidate) {
+                       return candidate->key == partition_key;
+                     }),
+      target_partitions.end());
+  target_partitions.push_back(partition_info);
+  partition_to_device_[partition_key] = target_device_id;
+  partition_info->owner_device_id = target_device_id;
+  partition_info->partition->dev_id = target_device_id;
+  return true;
+}
+
 void PartitionTracker::MarkDevicePartitionsFailed(int64_t device_id) {
   std::lock_guard<std::mutex> lock(mutex_);
 
@@ -250,6 +291,19 @@ std::vector<PartitionInfoPtr> PartitionTracker::GetIdlePartitions() const {
   return idle_partitions;
 }
 
+std::vector<PartitionInfoPtr> PartitionTracker::ClaimIdlePartitions() {
+  std::lock_guard<std::mutex> lock(mutex_);
+
+  std::vector<PartitionInfoPtr> claimed_partitions;
+  for (const auto& part_info : partitions_set_) {
+    if (part_info->state == PartitionState::IDLE) {
+      part_info->state = PartitionState::RUNNING;
+      claimed_partitions.push_back(part_info);
+    }
+  }
+  return claimed_partitions;
+}
+
 PartitionTracker::DeviceOidStats PartitionTracker::GetDeviceOidStats(
     int64_t device_id, int64_t oid) const {
   std::lock_guard<std::mutex> lock(mutex_);

csrc/backend/partition_tracker.h

@@ -68,6 +68,8 @@ class PartitionTracker {
                     int64_t oid, MatrixPartitionPtr partition);
   void RemovePartition(int64_t device_id, const std::string& partition_key);
   void RemovePartitionByKey(const std::string& partition_key);
+  bool ReassignPartitionToDevice(const std::string& partition_key,
+                                 int64_t target_device_id);
 
   // Mark all partitions owned by a device as failed (ownership removed)
   void MarkDevicePartitionsFailed(int64_t device_id);
@@ -85,6 +87,7 @@ class PartitionTracker {
   size_t GetDevicePartitionCount(int64_t device_id) const;
   bool HasPendingPartitions(int64_t device_id) const;
   std::vector<PartitionInfoPtr> GetIdlePartitions() const;
+  std::vector<PartitionInfoPtr> ClaimIdlePartitions();
 
   // Statistics for a specific OID on a device (for debugging)
   struct DeviceOidStats {