diff --git a/CHANGELOG.md b/CHANGELOG.md index 9aee319..01e06a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# NEXT + +- add `(lookup-file)` message to notify editor of file lookups +- fix zealous JSON escaping +- add `(open-base64)` command for binary data (@merv1n34k) +- add `-stream` flag for filesystem-independent editing (@merv1n34k) +- fix provider auto-detection short-circuit logic (@alvv-z) + # v0.2 Fri 6 Mar 19:09:31 JST 2026 Finally, the engine is independent of tectonic and the build does not need rust anymore: diff --git a/EDITOR-PROTOCOL.md b/EDITOR-PROTOCOL.md index 2eb25f7..9581f2d 100644 --- a/EDITOR-PROTOCOL.md +++ b/EDITOR-PROTOCOL.md @@ -128,6 +128,12 @@ Check the filesystem for changes. This will reload and reprocess any changed fil Asking the window manager to keep TeXpresso window above the others, or not. This can be convenient to keep a TeXpresso window floating on top of the editor. (`t` and `nil` are the closest approximation of "true" and "false" in emacs-sexp). +```scheme +(register "path") +``` + +Pre-register a filename that the editor will provide later via `open`. When the engine tries to read this file and it is not yet available, the driver pauses the engine and emits `request-file`. Once the editor sends `open` for the path, the engine resumes without restarting. + ```scheme (synctex-forward "path" line) ``` @@ -209,3 +215,17 @@ The paths are printed relative to the root file. They might be non-existent on t Right now, this is implemented by hooking into SyncTeX: - only text files are tracked (not graphics) - the indices printed are the SyncTex input indices; they should be attributed no other meaning than being monotonic and useful to detect backtracking occurrences + +### File lookups + +``` +(lookup-file kind status "path") +``` + +Output by TeXpresso when it tries to look up a file. +- `kind`: either `read` or `write`. +- `status`: either `successful`, `failed`, or `promised` if the editor had previously registered the file. +- `path`: the path to the file. + +If `status` is `promised`, document processing will be stuck until the editor fulfills the promise by sending a corresponding `(open "path" ...)` or `(open-base64 "path" ...)` command. +This can only happen if the editor had registered the path using `(register "path")`. diff --git a/Makefile b/Makefile index c435c28..ea74fc3 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,6 @@ all: @echo "# Or:" @echo "# build/texpresso -texlive test/simple.tex" @echo "# build/texpresso -tectonic test/simple.tex" - @echo "#" common: $(MAKE) -C src/common @@ -91,23 +90,31 @@ test-tectonic: build/texpresso-xetex -tectonic test/simple.tex rm simple.aux simple.log simple.xdv +TEST_TIMEOUT ?= 120 + test-open-base64: printf '(open-base64 "test/simple.tex" "%s")\n' "$$(base64 < test/simple.tex | tr -d '\n')" | \ - SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex + timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex test-texpresso: - SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex + timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/simple.tex test-texpresso-texlive: - SDL_VIDEODRIVER=dummy build/texpresso -texlive -test-initialize test/simple.tex + timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -texlive -test-initialize test/simple.tex test-texpresso-tectonic: - SDL_VIDEODRIVER=dummy build/texpresso -tectonic -test-initialize test/simple.tex + timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -tectonic -test-initialize test/simple.tex test-stream: - SDL_VIDEODRIVER=dummy build/texpresso -stream -test-initialize test/simple.tex + timeout $(TEST_TIMEOUT) env SDL_VIDEODRIVER=dummy build/texpresso -stream -test-initialize test/simple.tex test-stream-pipe: - test/test_stream.sh + timeout $(TEST_TIMEOUT) test/test_stream.sh + +test-request-file: + timeout $(TEST_TIMEOUT) bash test/test-request-file.sh + +test-register: + timeout $(TEST_TIMEOUT) bash test/test-register.sh -.PHONY: all dev clean config texpresso common texpresso-xetex re2c compile_commands.json fill-tectonic-cache test-texlive test-tectonic test-texpresso test-stream test-stream-pipe test-open-base64 +.PHONY: all dev clean config texpresso common texpresso-xetex re2c compile_commands.json fill-tectonic-cache test-texlive test-tectonic test-texpresso test-stream test-stream-pipe test-open-base64 test-request-file test-register diff --git a/emacs/texpresso.el b/emacs/texpresso.el index 979c7ad..af91d03 100644 --- a/emacs/texpresso.el +++ b/emacs/texpresso.el @@ -319,6 +319,8 @@ standard output. This function interprets one of these." ((eq tag 'input-file)) + ((eq tag 'lookup-file)) + (t (message "Unknown message in texpresso output: %S" expr))))) (defun texpresso--stdout-filter (process text) diff --git a/src/frontend/editor.c b/src/frontend/editor.c index 0c5694f..11c5408 100644 --- a/src/frontend/editor.c +++ b/src/frontend/editor.c @@ -281,6 +281,18 @@ bool editor_parse(fz_context *ctx, goto arity; *out = (struct editor_command){.tag = EDIT_INVERT, .invert = {}}; } + else if (strcmp(verb, "register") == 0) + { + if (len != 2) + goto arity; + val path = val_array_get(ctx, stack, command, 1); + if (!val_is_string(path)) + goto arguments; + *out = (struct editor_command){ + .tag = EDIT_REGISTER, + .reg = { .path = val_string(ctx, stack, path) }, + }; + } else { fprintf(stderr, "[command] unknown verb: %s\n", verb); @@ -306,7 +318,7 @@ static void output_json_string(FILE *f, const char *ptr, int len) unsigned char c = *ptr; if (c < 0x80) { - if (c < 0x32) + if (c < 0x20) { switch (c) { @@ -586,3 +598,41 @@ void editor_notify_file_opened(int index, const char *path, int len) case EDITOR_JSON: fprintf(stdout, "\"]\n"); break; } } + +void editor_notify_lookup(const char *path, + int len, + bool read, + enum EDITOR_LOOKUP_STATUS status) +{ + const char *kind = read ? "read" : "write"; + const char *status_msg; + + switch (status) + { + case LOOKUP_FAILED: + status_msg = "failed"; + case LOOKUP_PROMISED: + status_msg = "promised"; + case LOOKUP_SUCCESSFUL: + status_msg = "successful"; + default: + abort(); + } + + switch (protocol) + { + case EDITOR_SEXP: + fprintf(stdout, "(lookup-file %s %s \"", kind, status_msg); + break; + case EDITOR_JSON: + fprintf(stdout, "[\"lookup-file\", \"%s\", \"%s\", \"", kind, status_msg); + break; + } + + output_data_string(stdout, path, len); + switch (protocol) + { + case EDITOR_SEXP: fprintf(stdout, "\")\n"); break; + case EDITOR_JSON: fprintf(stdout, "\"]\n"); break; + } +} diff --git a/src/frontend/editor.h b/src/frontend/editor.h index 6aef93f..214ca6d 100644 --- a/src/frontend/editor.h +++ b/src/frontend/editor.h @@ -32,6 +32,7 @@ enum EDITOR_COMMAND EDIT_UNMAP_WINDOW, EDIT_CROP, EDIT_INVERT, + EDIT_REGISTER, }; struct editor_change @@ -114,6 +115,10 @@ struct editor_command struct { } invert; + + struct { + const char *path; + } reg; }; }; @@ -137,4 +142,16 @@ void editor_synctex(const char *dirname, const char *basename, int basename_len, void editor_reset_sync(void); void editor_notify_file_opened(int index, const char *path, int len); +enum EDITOR_LOOKUP_STATUS +{ + LOOKUP_SUCCESSFUL, + LOOKUP_FAILED, + LOOKUP_PROMISED, +}; + +void editor_notify_lookup(const char *path, + int len, + bool read, + enum EDITOR_LOOKUP_STATUS status); + #endif // EDITOR_H_ diff --git a/src/frontend/engine_tex.c b/src/frontend/engine_tex.c index 7651017..afaee97 100644 --- a/src/frontend/engine_tex.c +++ b/src/frontend/engine_tex.c @@ -88,6 +88,12 @@ struct tex_engine struct { int trace_len, offset, flush; } rollback; + + struct { + bool active; + query_t query; + char path[1024]; + } deferred; }; // Backtrackable process state & VFS representation @@ -191,6 +197,7 @@ static void prepare_process(fz_context *ctx, struct tex_engine *self) { if (self->process_count == 0) { + self->deferred.active = false; log_rollback(ctx, self->log, self->restart); self->process_count = 1; process_t *p = get_process(self); @@ -493,10 +500,30 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q) if (!fs_path) { e = filesystem_lookup_or_create(ctx, self->fs, q->open.path); - log_fileentry(ctx, self->log, e); - record_seen(self, e, INT_MAX, q->time); - a.tag = A_PASS; - channel_write_answer(self->c, p->fd, &a); + if (e->promised && !self->deferred.active) + { + // File was promised and is missing: notify the editor and wait + // for answer + self->deferred.active = true; + self->deferred.query = *q; + strncpy(self->deferred.path, q->open.path, + sizeof(self->deferred.path) - 1); + self->deferred.path[sizeof(self->deferred.path) - 1] = '\0'; + self->deferred.query.open.path = self->deferred.path; + editor_notify_lookup(q->open.path, strlen(q->open.path), true, + LOOKUP_PROMISED); + } + else + { + // File is missing: record this observation and mark the lookup + // as failed. + log_fileentry(ctx, self->log, e); + record_seen(self, e, INT_MAX, q->time); + a.tag = A_PASS; + editor_notify_lookup(q->open.path, strlen(q->open.path), true, + LOOKUP_FAILED); + channel_write_answer(self->c, p->fd, &a); + } break; } } @@ -529,6 +556,8 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q) log_fileentry(ctx, self->log, e); record_seen(self, e, INT_MAX, q->time); a.tag = A_PASS; + editor_notify_lookup(q->open.path, strlen(q->open.path), + q->tag == Q_OPRD, LOOKUP_FAILED); channel_write_answer(self->c, p->fd, &a); break; } @@ -620,6 +649,7 @@ static void answer_query(fz_context *ctx, struct tex_engine *self, query_t *q) int n = strlen(q->open.path); a.open.path_len = n; a.tag = A_OPEN; + editor_notify_lookup(q->open.path, n, q->tag == Q_OPRD, LOOKUP_SUCCESSFUL); memmove(channel_get_buffer(self->c, n), q->open.path, n); channel_write_answer(self->c, p->fd, &a); break; @@ -903,6 +933,7 @@ static void revert_trace(trace_entry_t *te) static void rollback_processes(fz_context *ctx, struct tex_engine *self, int reverted, int trace) { + self->deferred.active = false; fprintf( stderr, "rolling back to position %d\nbefore rollback: %d bytes of output\n", @@ -1064,6 +1095,20 @@ static bool engine_step(txp_engine *_self, fz_context *ctx, bool restart_if_need if (restart_if_needed) prepare_process(ctx, self); + if (self->deferred.active) + { + fileentry_t *e = filesystem_lookup(self->fs, self->deferred.path); + if (e && e->edit_data) + { + e->seen = -1; + self->deferred.active = false; + answer_query(ctx, self, &self->deferred.query); + channel_flush(self->c, get_process(self)->fd); + return 1; + } + return 0; + } + if (engine_get_status(_self) == DOC_RUNNING) { query_t q; @@ -1403,6 +1448,7 @@ txp_engine *txp_create_tex_engine(fz_context *ctx, self->stex = synctex_new(ctx); self->rollback.trace_len = NOT_IN_TRANSACTION; + self->deferred.active = false; return (txp_engine*)self; } diff --git a/src/frontend/main.c b/src/frontend/main.c index cbe436a..b7d54de 100644 --- a/src/frontend/main.c +++ b/src/frontend/main.c @@ -806,6 +806,7 @@ static void interpret_open(struct persistent_state *ps, flush_changes(ps, ui); int changed = -1; + bool had_edit_data = (e->edit_data != NULL); if (e->edit_data) { @@ -828,8 +829,13 @@ static void interpret_open(struct persistent_state *ps, if (changed >= 0) { - fprintf(stderr, "[command] open %s: changed offset is %d\n", path, changed); - send(notify_file_changes, ui->eng, ps->ctx, e, changed); + if (e->promised && !had_edit_data) + fprintf(stderr, "[command] open %s: resolving deferred query\n", path); + else + { + fprintf(stderr, "[command] open %s: changed offset is %d\n", path, changed); + send(notify_file_changes, ui->eng, ps->ctx, e, changed); + } } } @@ -904,6 +910,32 @@ SDL_SetWindowAlwaysOnTop(SDL_Window *window, SDL_bool state) #endif +static void interpret_register(struct persistent_state *ps, + ui_state *ui, + const char *path) +{ + if (path[0] == '/') + { + int go_up = 0; + path = relative_path(path, ps->doc_path, &go_up); + if (go_up > 0) + { + fprintf(stderr, "[command] register %s: file has a different root, skipping\n", path); + return; + } + } + + fileentry_t *e = send(find_file, ui->eng, ps->ctx, path); + if (!e) + { + fprintf(stderr, "[command] register %s: file not found, skipping\n", path); + return; + } + + e->promised = true; + fprintf(stderr, "[command] register %s: marked as promised\n", path); +} + static void interpret_command(struct persistent_state *ps, ui_state *ui, vstack *stack, @@ -1045,6 +1077,10 @@ static void interpret_command(struct persistent_state *ps, schedule_event(RENDER_EVENT); } break; + + case EDIT_REGISTER: + interpret_register(ps, ui, cmd.reg.path); + break; } } @@ -1461,7 +1497,9 @@ bool texpresso_main(struct persistent_state *ps) break; } } - if (ps->initialize_only) + if (ps->initialize_only && + (send(page_count, ui->eng) > 0 || + (send(get_status, ui->eng) == DOC_TERMINATED && stdin_eof))) { fprintf(stderr, "[info] Initialize mode: terminating engine process\n"); quit = 1; diff --git a/src/frontend/state.h b/src/frontend/state.h index 2613ae3..afbebcd 100644 --- a/src/frontend/state.h +++ b/src/frontend/state.h @@ -51,6 +51,7 @@ typedef struct fileentry_s { // State of the file in the text editor (or NULL if unedited) fz_buffer *edit_data; + bool promised; // State observed and/or produced by TeX process struct { diff --git a/test/missing-input.tex b/test/missing-input.tex new file mode 100644 index 0000000..4219056 --- /dev/null +++ b/test/missing-input.tex @@ -0,0 +1,5 @@ +\documentclass[12pt]{article} +\begin{document} +Hello. +\input{texpresso_ci_missing_file.tex} +\end{document} diff --git a/test/test-register.sh b/test/test-register.sh new file mode 100755 index 0000000..396f8bf --- /dev/null +++ b/test/test-register.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Test register + deferred Q_OPRD: editor pre-registers a file, +# engine defers the query, editor provides content, engine resumes. +set -e + +FIFO=$(mktemp -u /tmp/texpresso-fifo-XXXXXX) +OUTFILE=$(mktemp /tmp/texpresso-out-XXXXXX) +mkfifo "$FIFO" +trap 'rm -f "$FIFO" "$OUTFILE"; kill "$PID" 2>/dev/null || true' EXIT + +TARGET="texpresso_ci_missing_file.tex" + +SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/missing-input.tex \ + < "$FIFO" > "$OUTFILE" 2>/dev/null & +PID=$! + +exec 3>"$FIFO" + +# Pre-register the file before the engine asks for it +printf '(register "%s")\n' "$TARGET" >&3 + +# Wait for request-file (engine deferred Q_OPRD for the promised file) +while ! grep -q "request-file \"$TARGET\"" "$OUTFILE" 2>/dev/null; do + sleep 0.5 + if ! kill -0 "$PID" 2>/dev/null; then + echo "FAIL: texpresso exited before emitting request-file for $TARGET" + echo "stdout contents:" + cat "$OUTFILE" + exit 1 + fi +done + +echo "Got request-file for: $TARGET" + +# Provide the missing file content (resolves deferred query) +printf '(open "%s" "Included content.\\n")\n' "$TARGET" >&3 +exec 3>&- + +if wait "$PID"; then + echo "PASS: register test" +else + echo "FAIL: texpresso exited with error" + exit 1 +fi diff --git a/test/test-request-file.sh b/test/test-request-file.sh new file mode 100755 index 0000000..1c43073 --- /dev/null +++ b/test/test-request-file.sh @@ -0,0 +1,44 @@ +#!/bin/bash +# Test request-file: engine requests a missing file via Q_OPRL (non-blocking), +# test provides the file, engine restarts and processes it successfully. +set -e + +FIFO=$(mktemp -u /tmp/texpresso-fifo-XXXXXX) +OUTFILE=$(mktemp /tmp/texpresso-out-XXXXXX) +mkfifo "$FIFO" +trap 'rm -f "$FIFO" "$OUTFILE"; kill "$PID" 2>/dev/null || true' EXIT + +TARGET="texpresso_ci_missing_file.tex" + +# Start texpresso in background, reading stdin from FIFO, stdout to file +SDL_VIDEODRIVER=dummy build/texpresso -test-initialize test/missing-input.tex \ + < "$FIFO" > "$OUTFILE" 2>/dev/null & +PID=$! + +# Open FIFO for writing (unblocks texpresso's stdin) +exec 3>"$FIFO" + +# Wait for request-file for the target file (ignore .aux etc.) +while ! grep -q "request-file \"$TARGET\"" "$OUTFILE" 2>/dev/null; do + sleep 0.5 + if ! kill -0 "$PID" 2>/dev/null; then + echo "FAIL: texpresso exited before emitting request-file for $TARGET" + echo "stdout contents:" + cat "$OUTFILE" + exit 1 + fi +done + +echo "Got request-file for: $TARGET" + +# Provide the missing file content +printf '(open "%s" "Included content.\\n")\n' "$TARGET" >&3 +exec 3>&- + +# Wait for texpresso to finish (it exits after page_count > 0 in -test-initialize mode) +if wait "$PID"; then + echo "PASS: request-file test" +else + echo "FAIL: texpresso exited with error" + exit 1 +fi diff --git a/test/test_stream.sh b/test/test_stream.sh index 448a028..9b61417 100755 --- a/test/test_stream.sh +++ b/test/test_stream.sh @@ -10,10 +10,29 @@ if [ ! -f "$TEX_FILE" ]; then exit 1 fi +FIFO=$(mktemp -u /tmp/texpresso-fifo-XXXXXX) +mkfifo "$FIFO" +trap 'rm -f "$FIFO"; kill "$PID" 2>/dev/null || true' EXIT + # Escape content for sexp string: \ → \\, " → \", newline → \n, tab → \t CONTENT=$(sed -e 's/\\/\\\\/g' -e 's/"/\\"/g' -e 's/ /\\t/g' "$TEX_FILE" | \ awk '{ if (NR > 1) printf "\\n"; printf "%s", $0 }') SEXP="(open \"$TEX_FILE\" \"$CONTENT\")" -echo "$SEXP" | SDL_VIDEODRIVER=dummy build/texpresso -stream -test-initialize test/simple.tex +SDL_VIDEODRIVER=dummy build/texpresso -stream -test-initialize test/simple.tex \ + < "$FIFO" 2>/dev/null & +PID=$! + +# Keep FIFO open until texpresso processes the command +exec 3>"$FIFO" +printf '%s\n' "$SEXP" >&3 + +# Wait for texpresso to finish, then close +if wait "$PID"; then + echo "PASS: stream-pipe test" +else + echo "FAIL: texpresso exited with error" + exit 1 +fi +exec 3>&-