Stream mode: Phase 3

EDITOR-PROTOCOL.md

@@ -194,6 +194,18 @@ SyncTeX backward synchronisation: the user clicked on text produced by LaTeX sou
 Output by TeXpresso when the contents of its VFS has been lost. The editor should re-`open` any file before sharing `change`s.
 Not urgent: this notification is used mainly when debugging TeXpresso, it should not happen during normal use.
 
+### File requests
+
+```
+(request-file "path")
+```
+
+Output by TeXpresso when the engine needs a file that cannot be resolved locally. The resolution order is: driver VFS, disk, kpathsea/tectonic. Only after all these fail does TeXpresso emit `request-file`.
+
+This message is non-blocking: the engine continues (and may fail if the file is critical, e.g. `\input`). When the editor responds with an `open` command, TeXpresso stores the file and restarts the engine, which then finds the file in the VFS.
+
+The path is relative to the root document directory. Note that `request-file` may be emitted for auxiliary files (e.g. `.aux` on first run) that TeX handles gracefully when missing — the editor can safely ignore requests for files it cannot provide.
+
 ### Files used by the document
 
 ```

Makefile

@@ -110,4 +110,7 @@ test-stream:
 test-stream-pipe:
 	test/test_stream.sh
 
-.PHONY: all dev clean config texpresso common texpresso-xetex re2c compile_commands.json fill-tectonic-cache test-texlive test-tectonic test-texpresso test-stream test-stream-pipe test-open-base64
+test-request-file:
+	bash test/test-request-file.sh
+
+.PHONY: all dev clean config texpresso common texpresso-xetex re2c compile_commands.json fill-tectonic-cache test-texlive test-tectonic test-texpresso test-stream test-stream-pipe test-open-base64 test-request-file

src/engine/main/main.c

@@ -322,7 +322,24 @@ ttbc_input_handle_t *ttstub_input_open(const char *path,
   if (texpresso)
   {
     if (!f)
+    {
+      txp_file_id id = next_id();
+      char *ipath = txp_open_last_resort(texpresso, id, path,
+                                         kind_of_ttbc_format(format));
+      if (ipath)
+      {
+        strcpy(last_open, ipath);
+        free(ipath);
+        txp_input *input = calloc(1, sizeof(txp_input));
+        if (!input) abort();
+        alloc_id(id);
+        input->id = id;
+        input->file_size = -1;
+        input->generation = txp_generation(texpresso);
+        return txp_as_input(input);
+      }
       return NULL;
+    }
 
     txp_input_file *input = calloc(1, sizeof(txp_input_file));
     if (!input)

src/engine/main/texpresso_protocol.c

@@ -26,6 +26,7 @@ enum tag
   T_FORK = FOURCC('F', 'O', 'R', 'K'),
   T_GPIC = FOURCC('G', 'P', 'I', 'C'),
   T_OPRD = FOURCC('O', 'P', 'R', 'D'),
+  T_OPRL = FOURCC('O', 'P', 'R', 'L'),
   T_OPWR = FOURCC('O', 'P', 'W', 'R'),
   T_OPEN = FOURCC('O', 'P', 'E', 'N'),
   T_PASS = FOURCC('P', 'A', 'S', 'S'),
@@ -252,6 +253,39 @@ char *txp_open(txp_client *io,
   }
 }
 
+char *txp_open_last_resort(txp_client *io,
+                           txp_file_id file,
+                           const char *path,
+                           enum txp_file_kind kind)
+{
+  fprintf(stderr, "txp_open_last_resort(\"%s\")\n", path);
+  txp_io_send_tag(io, T_OPRL);
+  txp_io_send_u32(io, file);
+  txp_io_send_str(io, path);
+  txp_io_send_u32(io, kind);
+  enum tag t = txp_io_recv_tag(io);
+  switch (t)
+  {
+    case T_PASS:
+      return NULL;
+    case T_OPEN:
+    {
+      uint32_t size = txp_io_recv_u32(io);
+      char *buf = calloc(1, size + 1);
+      if (buf == NULL)
+      {
+        fprintf(stderr, "Cannot allocate filename (length: %d)\n", size);
+        exit(1);
+      }
+      read_exact(io->file, buf, size);
+      buf[size] = 0;
+      return buf;
+    }
+    default:
+      panic_tag(t);
+  }
+}
+
 size_t txp_read(txp_client *io,
                 txp_file_id file,
                 uint32_t pos,

src/engine/main/texpresso_protocol.h

@@ -70,6 +70,9 @@ void txp_flush(txp_client *client);
 // Open a file
 char *txp_open(txp_client *client, txp_file_id file, const char *path, enum txp_file_kind kind, enum txp_open_mode mode);
 
+// Open a file (last resort, after all local fallbacks failed)
+char *txp_open_last_resort(txp_client *client, txp_file_id file, const char *path, enum txp_file_kind kind);
+
 // Read from a file
 size_t txp_read(txp_client *client, txp_file_id file, uint32_t pos, void *buf, size_t len);
 

src/frontend/editor.c

@@ -586,3 +586,24 @@ void editor_notify_file_opened(int index, const char *path, int len)
     case EDITOR_JSON: fprintf(stdout, "\"]\n"); break;
   }
 }
+
+void editor_request_file(const char *path, int len)
+{
+  if (len == 0)
+    return;
+  switch (protocol)
+  {
+    case EDITOR_SEXP:
+      fprintf(stdout, "(request-file \"");
+      break;
+    case EDITOR_JSON:
+      fprintf(stdout, "[\"request-file\", \"");
+      break;
+  }
+  output_data_string(stdout, path, len);
+  switch (protocol)
+  {
+    case EDITOR_SEXP: fprintf(stdout, "\")\n"); break;
+    case EDITOR_JSON: fprintf(stdout, "\"]\n"); break;
+  }
+}

src/frontend/editor.h

@@ -136,5 +136,6 @@ void editor_flush(void);
 void editor_synctex(const char *dirname, const char *basename, int basename_len, int line, int column);
 void editor_reset_sync(void);
 void editor_notify_file_opened(int index, const char *path, int len);
+void editor_request_file(const char *path, int len);
 
 #endif  // EDITOR_H_

src/frontend/engine_tex.c

@@ -624,6 +624,20 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q)
       channel_write_answer(self->c, p->fd, &a);
       break;
     }
+    case Q_OPRL:
+    {
+      check_fid(q->open.fid);
+      filecell_t *cell = &self->st.table[q->open.fid];
+      if (cell->entry != NULL) mabort();
+
+      fileentry_t *e = filesystem_lookup_or_create(ctx, self->fs, q->open.path);
+      log_fileentry(ctx, self->log, e);
+      record_seen(self, e, INT_MAX, q->time);
+      a.tag = A_PASS;
+      channel_write_answer(self->c, p->fd, &a);
+      editor_request_file(q->open.path, strlen(q->open.path));
+      break;
+    }
     case Q_READ:
     {
       check_fid(q->read.fid);

src/frontend/main.c

@@ -788,12 +788,15 @@ static void interpret_open(struct persistent_state *ps,
                            const void *data,
                            int size)
 {
-  int go_up = 0;
-  path = relative_path(path, ps->doc_path, &go_up);
-  if (go_up > 0)
+  if (path[0] == '/')
   {
-    fprintf(stderr, "[command] open %s: file has a different root, skipping\n", path);
-    return;
+    int go_up = 0;
+    path = relative_path(path, ps->doc_path, &go_up);
+    if (go_up > 0)
+    {
+      fprintf(stderr, "[command] open %s: file has a different root, skipping\n", path);
+      return;
+    }
   }
 
   fileentry_t *e = send(find_file, ui->eng, ps->ctx, path);
@@ -1461,7 +1464,9 @@ bool texpresso_main(struct persistent_state *ps)
           break;
       }
     }
-    if (ps->initialize_only)
+    if (ps->initialize_only &&
+        (send(page_count, ui->eng) > 0 ||
+         (send(get_status, ui->eng) == DOC_TERMINATED && stdin_eof)))
     {
       fprintf(stderr, "[info] Initialize mode: terminating engine process\n");
       quit = 1;

src/frontend/sprotocol.c

@@ -209,6 +209,7 @@ const char *query_to_string(enum query q)
   switch (q)
   {
     CASE(Q,OPRD);
+    CASE(Q,OPRL);
     CASE(Q,OPWR);
     CASE(Q,READ);
     CASE(Q,APND);
@@ -389,6 +390,9 @@ void log_query(FILE *f, query_t *r)
     case Q_OPRD:
       fprintf(f, "OPRD(%d, \"%s\")\n", r->open.fid, r->open.path);
       return;
+    case Q_OPRL:
+      fprintf(f, "OPRL(%d, \"%s\")\n", r->open.fid, r->open.path);
+      return;
     case Q_OPWR:
       fprintf(f, "OPWR(%d, \"%s\")\n", r->open.fid, r->open.path);
       return;
@@ -471,6 +475,7 @@ bool channel_read_query(channel_t *t, int fd, query_t *r)
   switch (tag)
   {
     case Q_OPRD:
+    case Q_OPRL:
     case Q_OPWR:
       {
         r->open.fid = read_u32(t, fd);

src/frontend/sprotocol.h

@@ -51,6 +51,7 @@ typedef int file_id;
 
 enum query {
   Q_OPRD = PACK('O','P','R','D'),
+  Q_OPRL = PACK('O','P','R','L'),
   Q_OPWR = PACK('O','P','W','R'),
   Q_READ = PACK('R','E','A','D'),
   Q_APND = PACK('A','P','N','D'),

test/request-file.tex

@@ -0,0 +1,5 @@
+\documentclass[12pt]{article}
+\begin{document}
+Hello.
+\input{texpresso_ci_missing_file.tex}
+\end{document}

test/test-request-file.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+# Test request-file: engine requests a missing file via Q_OPRL (non-blocking),
+# test provides the file, engine restarts and processes it successfully.
+set -e
+
+FIFO=$(mktemp -u /tmp/texpresso-fifo-XXXXXX)
+OUTFILE=$(mktemp /tmp/texpresso-out-XXXXXX)
+mkfifo "$FIFO"
+trap 'rm -f "$FIFO" "$OUTFILE"; kill "$PID" 2>/dev/null || true' EXIT
+
+TARGET="texpresso_ci_missing_file.tex"
+
+# Start texpresso in background, reading stdin from FIFO, stdout to file
+SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/request-file.tex \
+  < "$FIFO" > "$OUTFILE" 2>/dev/null &
+PID=$!
+
+# Open FIFO for writing (unblocks texpresso's stdin)
+exec 3>"$FIFO"
+
+# Wait for request-file for the target file (ignore .aux etc.)
+TIMEOUT=120
+while ! grep -q "request-file \"$TARGET\"" "$OUTFILE" 2>/dev/null; do
+  sleep 0.5
+  TIMEOUT=$((TIMEOUT - 1))
+  if [ $TIMEOUT -le 0 ]; then
+    echo "FAIL: timeout waiting for request-file"
+    echo "stdout contents:"
+    cat "$OUTFILE"
+    exit 1
+  fi
+  if ! kill -0 "$PID" 2>/dev/null; then
+    echo "FAIL: texpresso exited before emitting request-file for $TARGET"
+    echo "stdout contents:"
+    cat "$OUTFILE"
+    exit 1
+  fi
+done
+
+echo "Got request-file for: $TARGET"
+
+# Provide the missing file content
+printf '(open "%s" "Included content.\\n")\n' "$TARGET" >&3
+exec 3>&-
+
+# Wait for texpresso to finish (it exits after page_count > 0 in -test-initialize mode)
+if wait "$PID"; then
+  echo "PASS: request-file test"
+else
+  echo "FAIL: texpresso exited with error"
+  exit 1
+fi