"Stream mode: Phase 4 finale" on top of lookup-file

CHANGELOG.md

@@ -1,3 +1,11 @@
+# NEXT
+
+- add `(lookup-file)` message to notify editor of file lookups
+- fix zealous JSON escaping
+- add `(open-base64)` command for binary data (@merv1n34k)
+- add `-stream` flag for filesystem-independent editing (@merv1n34k)
+- fix provider auto-detection short-circuit logic (@alvv-z)
+
 # v0.2 Fri  6 Mar 19:09:31 JST 2026
 
 Finally, the engine is independent of tectonic and the build does not need rust anymore:

EDITOR-PROTOCOL.md

@@ -128,6 +128,12 @@ Check the filesystem for changes. This will reload and reprocess any changed fil
 
 Asking the window manager to keep TeXpresso window above the others, or not. This can be convenient to keep a TeXpresso window floating on top of the editor. (`t` and `nil` are the closest approximation of "true" and "false" in emacs-sexp).
 
+```scheme
+(register "path")
+```
+
+Pre-register a filename that the editor will provide later via `open`. When the engine tries to read this file and it is not yet available, the driver pauses the engine and emits `request-file`. Once the editor sends `open` for the path, the engine resumes without restarting.
+
 ```scheme
 (synctex-forward "path" line)
 ```
@@ -209,3 +215,17 @@ The paths are printed relative to the root file. They might be non-existent on t
 Right now, this is implemented by hooking into SyncTeX:
 - only text files are tracked (not graphics)
 - the indices printed are the SyncTex input indices; they should be attributed no other meaning than being monotonic and useful to detect backtracking occurrences
+
+### File lookups
+
+```
+(lookup-file kind status "path")
+```
+
+Output by TeXpresso when it tries to look up a file.
+- `kind`: either `read` or `write`.
+- `status`: either `successful`, `failed`, or `promised` if the editor had previously registered the file.
+- `path`: the path to the file.
+
+If `status` is `promised`, document processing will be stuck until the editor fulfills the promise by sending a corresponding `(open "path" ...)` or `(open-base64 "path" ...)` command.
+This can only happen if the editor had registered the path using `(register "path")`.

Makefile

@@ -18,7 +18,6 @@ all:
 	@echo "# Or:"
 	@echo "#   build/texpresso -texlive test/simple.tex"
 	@echo "#   build/texpresso -tectonic test/simple.tex"
-	@echo "#"
 
 common:
 	$(MAKE) -C src/common
@@ -91,23 +90,31 @@ test-tectonic:
 	build/texpresso-xetex -tectonic test/simple.tex
 	rm simple.aux simple.log simple.xdv
 
+TEST_TIMEOUT ?= 120
+
 test-open-base64:
 	printf '(open-base64 "test/simple.tex" "%s")\n' "$$(base64 < test/simple.tex | tr -d '\n')" | \
-	SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex
+	timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex
 
 test-texpresso:
-	SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex
+	timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex
 
 test-texpresso-texlive:
-	SDL_VIDEODRIVER=dummy build/texpresso -texlive -test-initialize test/simple.tex
+	timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -texlive -test-initialize test/simple.tex
 
 test-texpresso-tectonic:
-	SDL_VIDEODRIVER=dummy build/texpresso -tectonic -test-initialize test/simple.tex
+	timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -tectonic -test-initialize test/simple.tex
 
 test-stream:
-	SDL_VIDEODRIVER=dummy build/texpresso -stream -test-initialize test/simple.tex
+	timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -stream -test-initialize test/simple.tex
 
 test-stream-pipe:
-	test/test_stream.sh
+	timeout $(TEST_TIMEOUT) test/test_stream.sh
+
+test-request-file:
+	timeout $(TEST_TIMEOUT) bash test/test-request-file.sh
+
+test-register:
+	timeout $(TEST_TIMEOUT) bash test/test-register.sh
 
-.PHONY: all dev clean config texpresso common texpresso-xetex re2c compile_commands.json fill-tectonic-cache test-texlive test-tectonic test-texpresso test-stream test-stream-pipe test-open-base64
+.PHONY: all dev clean config texpresso common texpresso-xetex re2c compile_commands.json fill-tectonic-cache test-texlive test-tectonic test-texpresso test-stream test-stream-pipe test-open-base64 test-request-file test-register

emacs/texpresso.el

@@ -319,6 +319,8 @@ standard output. This function interprets one of these."
 
      ((eq tag 'input-file))
 
+     ((eq tag 'lookup-file))
+
      (t (message "Unknown message in texpresso output: %S" expr)))))
 
 (defun texpresso--stdout-filter (process text)

src/frontend/editor.c

@@ -281,6 +281,18 @@ bool editor_parse(fz_context *ctx,
       goto arity;
     *out = (struct editor_command){.tag = EDIT_INVERT, .invert = {}};
   }
+  else if (strcmp(verb, "register") == 0)
+  {
+    if (len != 2)
+      goto arity;
+    val path = val_array_get(ctx, stack, command, 1);
+    if (!val_is_string(path))
+      goto arguments;
+    *out = (struct editor_command){
+        .tag = EDIT_REGISTER,
+        .reg = { .path = val_string(ctx, stack, path) },
+    };
+  }
   else
   {
     fprintf(stderr, "[command] unknown verb: %s\n", verb);
@@ -306,7 +318,7 @@ static void output_json_string(FILE *f, const char *ptr, int len)
     unsigned char c = *ptr;
     if (c < 0x80)
     {
-      if (c < 0x32)
+      if (c < 0x20)
       {
         switch (c)
         {
@@ -586,3 +598,41 @@ void editor_notify_file_opened(int index, const char *path, int len)
     case EDITOR_JSON: fprintf(stdout, "\"]\n"); break;
   }
 }
+
+void editor_notify_lookup(const char *path,
+                          int len,
+                          bool read,
+                          enum EDITOR_LOOKUP_STATUS status)
+{
+  const char *kind = read ? "read" : "write";
+  const char *status_msg;
+
+  switch (status)
+  {
+    case LOOKUP_FAILED:
+      status_msg = "failed";
+    case LOOKUP_PROMISED:
+      status_msg = "promised";
+    case LOOKUP_SUCCESSFUL:
+      status_msg = "successful";
+    default:
+      abort();
+  }
+
+  switch (protocol)
+  {
+    case EDITOR_SEXP:
+      fprintf(stdout, "(lookup-file %s %s \"", kind, status_msg);
+      break;
+    case EDITOR_JSON:
+      fprintf(stdout, "[\"lookup-file\", \"%s\", \"%s\", \"", kind, status_msg);
+      break;
+  }
+
+  output_data_string(stdout, path, len);
+  switch (protocol)
+  {
+    case EDITOR_SEXP: fprintf(stdout, "\")\n"); break;
+    case EDITOR_JSON: fprintf(stdout, "\"]\n"); break;
+  }
+}

src/frontend/editor.h

@@ -32,6 +32,7 @@ enum EDITOR_COMMAND
   EDIT_UNMAP_WINDOW,
   EDIT_CROP,
   EDIT_INVERT,
+  EDIT_REGISTER,
 };
 
 struct editor_change
@@ -114,6 +115,10 @@ struct editor_command
 
     struct {
     } invert;
+
+    struct {
+      const char *path;
+    } reg;
   };
 };
 
@@ -137,4 +142,16 @@ void editor_synctex(const char *dirname, const char *basename, int basename_len,
 void editor_reset_sync(void);
 void editor_notify_file_opened(int index, const char *path, int len);
 
+enum EDITOR_LOOKUP_STATUS
+{
+  LOOKUP_SUCCESSFUL,
+  LOOKUP_FAILED,
+  LOOKUP_PROMISED,
+};
+
+void editor_notify_lookup(const char *path,
+                          int len,
+                          bool read,
+                          enum EDITOR_LOOKUP_STATUS status);
+
 #endif  // EDITOR_H_

src/frontend/engine_tex.c

@@ -88,6 +88,12 @@ struct tex_engine
   struct {
     int trace_len, offset, flush;
   } rollback;
+
+  struct {
+    bool active;
+    query_t query;
+    char path[1024];
+  } deferred;
 };
 
 // Backtrackable process state & VFS representation
@@ -191,6 +197,7 @@ static void prepare_process(fz_context *ctx, struct tex_engine *self)
 {
   if (self->process_count == 0)
   {
+    self->deferred.active = false;
     log_rollback(ctx, self->log, self->restart);
     self->process_count = 1;
     process_t *p = get_process(self);
@@ -493,10 +500,30 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q)
           if (!fs_path)
           {
             e = filesystem_lookup_or_create(ctx, self->fs, q->open.path);
-            log_fileentry(ctx, self->log, e);
-            record_seen(self, e, INT_MAX, q->time);
-            a.tag = A_PASS;
-            channel_write_answer(self->c, p->fd, &a);
+            if (e->promised && !self->deferred.active)
+            {
+              // File was promised and is missing: notify the editor and wait
+              // for answer
+              self->deferred.active = true;
+              self->deferred.query = *q;
+              strncpy(self->deferred.path, q->open.path,
+                      sizeof(self->deferred.path) - 1);
+              self->deferred.path[sizeof(self->deferred.path) - 1] = '\0';
+              self->deferred.query.open.path = self->deferred.path;
+              editor_notify_lookup(q->open.path, strlen(q->open.path), true,
+                                   LOOKUP_PROMISED);
+            }
+            else
+            {
+              // File is missing: record this observation and mark the lookup
+              // as failed.
+              log_fileentry(ctx, self->log, e);
+              record_seen(self, e, INT_MAX, q->time);
+              a.tag = A_PASS;
+              editor_notify_lookup(q->open.path, strlen(q->open.path), true,
+                                   LOOKUP_FAILED);
+              channel_write_answer(self->c, p->fd, &a);
+            }
             break;
           }
         }
@@ -529,6 +556,8 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q)
                 log_fileentry(ctx, self->log, e);
                 record_seen(self, e, INT_MAX, q->time);
                 a.tag = A_PASS;
+                editor_notify_lookup(q->open.path, strlen(q->open.path),
+                                     q->tag == Q_OPRD, LOOKUP_FAILED);
                 channel_write_answer(self->c, p->fd, &a);
                 break;
               }
@@ -620,6 +649,7 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q)
       int n = strlen(q->open.path);
       a.open.path_len = n;
       a.tag = A_OPEN;
+      editor_notify_lookup(q->open.path, n, q->tag == Q_OPRD, LOOKUP_SUCCESSFUL);
       memmove(channel_get_buffer(self->c, n), q->open.path, n);
       channel_write_answer(self->c, p->fd, &a);
       break;
@@ -903,6 +933,7 @@ static void revert_trace(trace_entry_t *te)
 
 static void rollback_processes(fz_context *ctx, struct tex_engine *self, int reverted, int trace)
 {
+  self->deferred.active = false;
   fprintf(
     stderr,
     "rolling back to position %d\nbefore rollback: %d bytes of output\n",
@@ -1064,6 +1095,20 @@ static bool engine_step(txp_engine *_self, fz_context *ctx, bool restart_if_need
   if (restart_if_needed)
     prepare_process(ctx, self);
 
+  if (self->deferred.active)
+  {
+    fileentry_t *e = filesystem_lookup(self->fs, self->deferred.path);
+    if (e && e->edit_data)
+    {
+      e->seen = -1;
+      self->deferred.active = false;
+      answer_query(ctx, self, &self->deferred.query);
+      channel_flush(self->c, get_process(self)->fd);
+      return 1;
+    }
+    return 0;
+  }
+
   if (engine_get_status(_self) == DOC_RUNNING)
   {
     query_t q;
@@ -1403,6 +1448,7 @@ txp_engine *txp_create_tex_engine(fz_context *ctx,
 
   self->stex = synctex_new(ctx);
   self->rollback.trace_len = NOT_IN_TRANSACTION;
+  self->deferred.active = false;
 
   return (txp_engine*)self;
 }

src/frontend/main.c

@@ -806,6 +806,7 @@ static void interpret_open(struct persistent_state *ps,
   flush_changes(ps, ui);
 
   int changed = -1;
+  bool had_edit_data = (e->edit_data != NULL);
 
   if (e->edit_data)
   {
@@ -828,8 +829,13 @@ static void interpret_open(struct persistent_state *ps,
 
   if (changed >= 0)
   {
-    fprintf(stderr, "[command] open %s: changed offset is %d\n", path, changed);
-    send(notify_file_changes, ui->eng, ps->ctx, e, changed);
+    if (e->promised && !had_edit_data)
+      fprintf(stderr, "[command] open %s: resolving deferred query\n", path);
+    else
+    {
+      fprintf(stderr, "[command] open %s: changed offset is %d\n", path, changed);
+      send(notify_file_changes, ui->eng, ps->ctx, e, changed);
+    }
   }
 }
 
@@ -904,6 +910,32 @@ SDL_SetWindowAlwaysOnTop(SDL_Window *window, SDL_bool state)
 #endif
 
 
+static void interpret_register(struct persistent_state *ps,
+                               ui_state *ui,
+                               const char *path)
+{
+  if (path[0] == '/')
+  {
+    int go_up = 0;
+    path = relative_path(path, ps->doc_path, &go_up);
+    if (go_up > 0)
+    {
+      fprintf(stderr, "[command] register %s: file has a different root, skipping\n", path);
+      return;
+    }
+  }
+
+  fileentry_t *e = send(find_file, ui->eng, ps->ctx, path);
+  if (!e)
+  {
+    fprintf(stderr, "[command] register %s: file not found, skipping\n", path);
+    return;
+  }
+
+  e->promised = true;
+  fprintf(stderr, "[command] register %s: marked as promised\n", path);
+}
+
 static void interpret_command(struct persistent_state *ps,
                               ui_state *ui,
                               vstack *stack,
@@ -1045,6 +1077,10 @@ static void interpret_command(struct persistent_state *ps,
       schedule_event(RENDER_EVENT);
     }
     break;
+
+    case EDIT_REGISTER:
+      interpret_register(ps, ui, cmd.reg.path);
+      break;
   }
 }
 
@@ -1461,7 +1497,9 @@ bool texpresso_main(struct persistent_state *ps)
           break;
       }
     }
-    if (ps->initialize_only)
+    if (ps->initialize_only &&
+        (send(page_count, ui->eng) > 0 ||
+         (send(get_status, ui->eng) == DOC_TERMINATED && stdin_eof)))
     {
       fprintf(stderr, "[info] Initialize mode: terminating engine process\n");
       quit = 1;