rpc: one tensor read per decode step and asynchronous split copies

.gitignore

@@ -153,3 +153,4 @@ a.out.*
 
 AGENTS.local.md
 .pi/SYSTEM.md
+bench/

ggml/src/ggml-backend-impl.h

@@ -8,7 +8,10 @@
 extern "C" {
 #endif
 
-    #define GGML_BACKEND_API_VERSION 2
+    // 3: ggml_backend_i gained cpy_tensor_from_async. A dynamically loaded backend compiled
+    //    against version 2 supplies a struct one member short, so the version has to move with it
+    //    for the load-time check to reject that pairing instead of reading past the end.
+    #define GGML_BACKEND_API_VERSION 3
 
     //
     // Backend buffer type
@@ -138,6 +141,15 @@ extern "C" {
 
         // (optional) sort/optimize the nodes in the graph
         void                      (*graph_optimize)    (ggml_backend_t backend, struct ggml_cgraph * cgraph);
+
+        // (optional) copy a tensor of this backend into another backend's tensor, driven from the
+        // source side. Kept separate from cpy_tensor_async because every existing implementation
+        // of that is written as a destination side handler: it casts backend_dst to its own
+        // context before deciding anything, so dispatching a source side call there would
+        // reinterpret a foreign backend_dst, or dereference a null one. Implement this only if
+        // the source role is genuinely supported. Appended last so backends that list the
+        // members positionally are unaffected.
+        bool (*cpy_tensor_from_async)(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst);
     };
 
     struct ggml_backend {

ggml/src/ggml-backend.cpp

@@ -499,6 +499,27 @@ void ggml_backend_tensor_copy(const struct ggml_tensor * src, struct ggml_tensor
     }
 }
 
+// src is only asked when its implementation differs: same-type backends share one implementation that already declined.
+static bool ggml_backend_cpy_tensor_async_impl(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
+    if (backend_dst != NULL && backend_dst->iface.cpy_tensor_async != NULL) {
+        if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
+            return true;
+        }
+    }
+
+    // backend_dst must be a real backend before anything is delegated to the source. An
+    // implementation of the source role has to inspect the destination to decide whether it can
+    // help, so handing it a null destination pushes that dereference into every implementer.
+    // Guarding here keeps the invariant in one place instead of depending on all of them.
+    if (backend_src != NULL && backend_dst != NULL && backend_src->iface.cpy_tensor_from_async != NULL) {
+        if (backend_src->iface.cpy_tensor_from_async(backend_src, backend_dst, src, dst)) {
+            return true;
+        }
+    }
+
+    return false;
+}
+
 void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t backend_dst, const struct ggml_tensor * src, struct ggml_tensor * dst) {
     GGML_ASSERT(ggml_are_same_layout(src, dst) && "cannot copy tensors with different layouts");
 
@@ -507,10 +528,8 @@ void ggml_backend_tensor_copy_async(ggml_backend_t backend_src, ggml_backend_t b
     }
 
     GGML_ASSERT(backend_dst);
-    if (backend_dst->iface.cpy_tensor_async != NULL) {
-        if (backend_dst->iface.cpy_tensor_async(backend_src, backend_dst, src, dst)) {
-            return;
-        }
+    if (ggml_backend_cpy_tensor_async_impl(backend_src, backend_dst, src, dst)) {
+        return;
     }
 
     // an async copy would normally happen after all the queued operations on both backends are completed
@@ -1728,7 +1747,7 @@ static enum ggml_status ggml_backend_sched_compute_splits(ggml_backend_sched_t s
                 } else {
                     // try async copy, but if not possible, we can still use a sync copy without synchronizing the dst backend, since we handle the synchronization here with multiple copies and events
                     // TODO: add public function to facilitate this, since applications do not have direct access to the backend interface
-                    if (!split_backend->iface.cpy_tensor_async || !split_backend->iface.cpy_tensor_async(input_backend, split_backend, input, input_cpy)) {
+                    if (!ggml_backend_cpy_tensor_async_impl(input_backend, split_backend, input, input_cpy)) {
                         ggml_backend_synchronize(input_backend);
                         if (sched->events[split_backend_id][sched->cur_copy] != NULL) {
                             ggml_backend_event_synchronize(sched->events[split_backend_id][sched->cur_copy]);