core+lsp: one registration seam; the LSP builtin table is generated (#742)

.github/workflows/ci.yml

@@ -282,6 +282,10 @@ jobs:
         name: Embedding API smoke test (linked against the amalgamation)
         run: make embed-smoke
 
+      - if: needs.scope.outputs.code == 'true'
+        name: Embedding API smoke, gfx variant (one registration seam, #742)
+        run: make embed-smoke-gfx
+
       - if: needs.scope.outputs.code == 'true'
         name: Static embed library (make lib)
         run: make lib

.gitignore

@@ -48,4 +48,5 @@ tests/tmp_mod_373*.eigs
 build/
 # Generated LSP stdlib index (make lsp / build.sh lsp, #590)
 src/lsp_stdlib_index.h
+src/lsp_builtin_index.h
 src/eigsdap

CHANGELOG.md

@@ -6,6 +6,32 @@ All notable changes to EigenScript are documented here.
 
 ### Fixed
 
+- **The global env is composed through one registration seam, and the
+  LSP's builtin table is generated from it (#742).** gfx and store were
+  registered by hand at each entry point — gfx only in `main.c`, so a
+  `make gfx` build used through the embedding API had **no gfx builtins
+  at all** (`gfx_open` raised `undefined variable`). `register_builtins`
+  now composes store (always) and gfx (when built) itself, the five
+  hand-call sites are gone, and `make embed-smoke-gfx` (new, in the
+  extensions CI job) pins the embed+gfx leg that was broken. Store and
+  gfx thereby move inside the `[0, g_builtin_binding_count)` builtin
+  band, so `eigs_is_registered_builtin` now correctly claims them. On
+  the tooling half: `eigenlsp`'s hand-written `builtin_docs` table — the
+  repo's cleanest natural experiment in gating, at 68 of 235 builtins
+  (29%) plus a phantom `exec` while the two *gated* registries sat at
+  zero drift — is replaced by `src/lsp_builtin_index.h`, generated by
+  `tools/gen_lsp_builtin_index.sh` from the registration seams +
+  `ext_names.h` (names) and the signature comments above each C
+  definition (hover detail): 336 names, set-identical to `--api`'s
+  registry, 202 with signature detail, the rest falling back to a
+  BUILTINS.md pointer rather than papering over the missing comments. A
+  build artifact like `lsp_stdlib_index.h`, never committed. Completing
+  the table surfaced the builtin/stdlib name collision the old table
+  hid by omission (`mean` is both a tensor builtin and `stats.mean`):
+  hover now treats a dotted token as a member access — only the
+  module-qualified stdlib branch may claim it, so `stats.mean` hovers as
+  the stats function and bare `mean` as the builtin.
+
 - **The GC's two lockstep dispatches are generated from one table, and
   `CallFrame` init/release have one definition each (#743).** The
   lockstep between `GC_FOR_EACH_CHILD` and `gc_clear_node` was enforced

Makefile

@@ -46,7 +46,7 @@ LSP_BINARY  := $(SRC_DIR)/eigenlsp
 DAP_SOURCES := $(SRC_DIR)/eigsdap.c $(SRC_DIR)/tape_read.c $(filter-out $(CLI_ONLY),$(SOURCES))
 DAP_BINARY  := $(SRC_DIR)/eigsdap
 
-.PHONY: all build full http net gfx zlib lib amalgamation tsan test install install-gfx clean coverage coverage-clean fuzz fuzz-run lsp dap jit-smoke embed-smoke asan valgrind pgo poison freestanding-check freestanding-libc-diff asan-http print-%
+.PHONY: all build full http net gfx zlib lib amalgamation tsan test install install-gfx clean coverage coverage-clean fuzz fuzz-run lsp dap jit-smoke embed-smoke embed-smoke-gfx asan valgrind pgo poison freestanding-check freestanding-libc-diff asan-http print-%
 
 # ---- Per-variant objdir engine (#740) -------------------------------------
 # The engine's rules are defined before `all`, so pin the default goal.
@@ -211,7 +211,15 @@ install: build lsp dap
 $(SRC_DIR)/lsp_stdlib_index.h: $(wildcard lib/*.eigs) tools/gen_lsp_stdlib_index.sh
 	bash tools/gen_lsp_stdlib_index.sh
 
-lsp: $(SRC_DIR)/lsp_stdlib_index.h
+# Builtin half of the same idea (#742): names from the registration seams +
+# ext_names.h, hover detail from the signature comments. Also a build
+# artifact, never committed.
+$(SRC_DIR)/lsp_builtin_index.h: $(SRC_DIR)/builtins.c $(SRC_DIR)/builtins_host.c \
+		$(SRC_DIR)/hash.c $(SRC_DIR)/ext_store.c $(SRC_DIR)/ext_names.h \
+		tools/gen_lsp_builtin_index.sh
+	bash tools/gen_lsp_builtin_index.sh
+
+lsp: $(SRC_DIR)/lsp_stdlib_index.h $(SRC_DIR)/lsp_builtin_index.h
 	$(CC) $(CFLAGS) -o $(LSP_BINARY) $(LSP_SOURCES) \
 		-DEIGENSCRIPT_EXT_HTTP=0 \
 		-DEIGENSCRIPT_EXT_MODEL=0 \
@@ -261,6 +269,16 @@ embed-smoke: amalgamation
 		-lm -lpthread
 	/tmp/embed_smoke
 
+# Same smoke against the gfx variant's objects: pins that the embed API's
+# env is composed by the ONE registration seam (#742 — pre-fix, only the
+# CLI registered gfx, so this exact link had no gfx builtins). Registration
+# needs no SDL init, so this runs headless.
+embed-smoke-gfx: gfx
+	$(CC) $(FLAGS_gfx) -o /tmp/embed_smoke_gfx $(SRC_DIR)/embed_smoke.c \
+		$(filter-out build/gfx/main.o,$(wildcard build/gfx/*.o)) \
+		-lm -lpthread $(LIBS_gfx)
+	/tmp/embed_smoke_gfx
+
 # AddressSanitizer + UndefinedBehaviorSanitizer build. Catches
 # use-after-free, buffer overflow, leaks, and undefined behavior that
 # the normal -O2 build silently tolerates. ~2x slower; for testing only.

build.sh

@@ -45,6 +45,7 @@ if [ "$1" = "lsp" ]; then
     # build.sh. The stdlib index header it includes (#590) is generated
     # from lib/ first — same rule as the Makefile lsp target.
     bash ../tools/gen_lsp_stdlib_index.sh
+    bash ../tools/gen_lsp_builtin_index.sh   # builtin half, #742 — same rule
     LSP_SOURCES=" $SOURCES "
     for u in $CLI_ONLY; do LSP_SOURCES="${LSP_SOURCES/ $u / }"; done
     LSP_SOURCES="$LSP_SOURCES eigenlsp.c"

src/builtins.c

@@ -5887,6 +5887,18 @@ void register_builtins(Env *env) {
     register_model_builtins(env);
 #endif
 
+#if EIGENSCRIPT_EXT_GFX
+    register_gfx_builtins(env);
+#endif
+
+    /* EigenStore — always compiled (ext_store.c; freestanding builds get
+     * its linkable no-op). gfx and store used to be registered by hand at
+     * every entry point (#742): main.c only for gfx, so `make gfx` used
+     * through the embed API had NO gfx builtins. Every entry point now
+     * composes the global env through this one seam — new registrars go
+     * HERE, never at a call site. */
+    register_store_builtins(env);
+
 #if EIGS_BORROW_GUARD
     /* #548 guard self-test hook — see builtin_borrow_guard_selftest. */
     if (getenv("EIGS_BORROW_GUARD_SELFTEST"))

src/eigenlsp.c

@@ -20,80 +20,16 @@ extern const char* tok_type_name(TokType t);
 extern void free_tokenlist(TokenList *tl);
 
 /* ================================================================
- * BUILTIN DOCUMENTATION TABLE
- * ================================================================ */
-
-static const char *builtin_docs[][2] = {
-    {"print", "print of value -- write value to stdout"},
-    {"len", "len of value -- length of string, list, or dict"},
-    {"append", "append of [list, value] -- add element to end of list"},
-    {"type", "type of value -- returns type name (\"num\", \"str\", \"list\", \"dict\", \"fn\", \"null\")"},
-    {"range", "range of n -- list [0, 1, ..., n-1]; range of [start, end] or [start, end, step]"},
-    {"str", "str of value -- convert to string"},
-    {"num", "num of value -- convert to number"},
-    {"keys", "keys of dict -- list of key names"},
-    {"values", "values of dict -- list of values"},
-    {"has_key", "has_key of [dict, key] -- 1 if key exists, 0 otherwise"},
-    {"dict_set", "dict_set of [dict, key, value] -- set key in dict"},
-    {"floor", "floor of n -- round down"},
-    {"ceil", "ceil of n -- round up"},
-    {"round", "round of n -- round to nearest integer"},
-    {"abs", "abs of n -- absolute value"},
-    {"min", "min of [a, b] -- smaller value"},
-    {"max", "max of [a, b] -- larger value"},
-    {"sqrt", "sqrt of n -- square root"},
-    {"sin", "sin of n -- sine"},
-    {"cos", "cos of n -- cosine"},
-    {"split", "split of [string, delimiter] -- split string into list"},
-    {"scan_ints", "scan_ints of text or [text, comment_marker] -- scan whitespace-delimited signed integers"},
-    {"scan_tokens", "scan_tokens of text or [text, comment_marker] -- scan whitespace-delimited token spans"},
-    {"scan_int_tokens", "scan_int_tokens of text or [text, comment_marker] -- token spans with integer classification"},
-    {"join", "join of [list, separator] -- join list into string"},
-    {"text_builder_new", "text_builder_new of null -- create a native growable text builder"},
-    {"text_builder_append", "text_builder_append of [builder, value] -- append value as text"},
-    {"text_builder_append_line", "text_builder_append_line of [builder, value] -- append value and newline"},
-    {"text_builder_extend", "text_builder_extend of [builder, values] -- append each list item as text"},
-    {"text_builder_part_count", "text_builder_part_count of builder -- number of appended parts"},
-    {"text_builder_clear", "text_builder_clear of builder -- clear builder contents"},
-    {"text_builder_to_string", "text_builder_to_string of builder -- render builder text"},
-    {"trim", "trim of string -- strip whitespace"},
-    {"contains", "contains of [string, substring] -- 1 if found"},
-    {"substr", "substr of [string, start, length] -- extract substring"},
-    {"index_of", "index_of of [string, substring] -- first index or -1"},
-    {"str_replace", "str_replace of [string, old, new] -- replace all occurrences"},
-    {"read_text", "read_text of path -- read file as string"},
-    {"read_line", "read_line of null -- read next line from stdin (null at EOF)"},
-    {"write_text", "write_text of [path, text] -- write string to file"},
-    {"file_exists", "file_exists of path -- 1 if file exists"},
-    {"is_dir", "is_dir of path -- 1 if path is a directory"},
-    {"json_encode", "json_encode of value -- encode as JSON string"},
-    {"json_decode", "json_decode of string -- parse JSON to value"},
-    {"spawn", "spawn of fn -- create thread running function, returns handle"},
-    {"thread_join", "thread_join of handle -- block until thread completes, returns result"},
-    {"channel", "channel of null -- create bounded message channel"},
-    {"send", "send of [channel, value] -- send value into channel"},
-    {"recv", "recv of channel -- receive value from channel"},
-    {"store_open", "store_open of path -- open/create EigenStore database"},
-    {"store_put", "store_put of [db, collection, record] -- insert record, returns key"},
-    {"store_get", "store_get of [db, collection, key] -- get record by key"},
-    {"store_query", "store_query of [db, collection] -- get all records"},
-    {"store_close", "store_close of db -- close database"},
-    {"assert", "assert of value -- abort if value is falsy"},
-    {"observe", "observe of var -- snapshot entropy report"},
-    {"report", "report of var -- return entropy state as string"},
-    {"sort", "sort of list -- sort list in ascending order"},
-    {"reverse", "reverse of list -- reverse list"},
-    {"map", "map of [fn, list] -- apply fn to each element"},
-    {"filter", "filter of [fn, list] -- keep elements where fn returns truthy"},
-    {"reduce", "reduce of [fn, initial, list] -- fold list with fn"},
-    {"exec", "exec of command -- run shell command, return stdout"},
-    {"exit", "exit of code -- exit process with code"},
-    {"time", "time of null -- seconds since epoch"},
-    {"sleep", "sleep of seconds -- pause execution"},
-    {"random", "random of null -- random float in [0,1)"},
-    {"eval", "eval of string -- evaluate EigenScript code string"},
-    {NULL, NULL}
-};
+ * BUILTIN DOCUMENTATION TABLE — generated (#742)
+ * ================================================================
+ * builtin_docs[][2] comes from tools/gen_lsp_builtin_index.sh: names from
+ * the registration seams + ext_names.h, hover detail from the signature
+ * comment above each C definition. The hand-written table this replaced
+ * was the repo's cleanest natural experiment in gating: the two gated
+ * registries (SANDBOX_ALLOW, the docs) sat at zero drift while this one —
+ * ungated — fell to 68 of 235 builtins plus a phantom `exec`. A build
+ * artifact, not committed; the Makefile lsp target regenerates it. */
+#include "lsp_builtin_index.h"
 
 /* ---- Keyword descriptions for hover ---- */
 static const char *keyword_docs[][2] = {
@@ -1071,16 +1007,27 @@ static void handle_hover(int id, const char *params) {
     const char *hover_text = NULL;
     char hover_buf[1024];
 
+    /* A dotted token (`x.name`) is a member access — never a bare builtin,
+     * keyword, or document symbol, so those lookups are skipped for it and
+     * only the module-qualified stdlib branch below may claim it. Became
+     * load-bearing with the generated builtin table (#742): every builtin
+     * is in builtin_docs now, so `stats.mean` would otherwise hover as the
+     * tensor builtin `mean` — the exact builtin/stdlib name collision the
+     * old 29%-complete hand table hid by omission. */
+    ptrdiff_t ti = tok - doc->tokens.tokens;
+    int dotted = (ti >= 1 && doc->tokens.tokens[ti - 1].type == TOK_DOT);
+
     /* Check builtins */
-    for (int i = 0; builtin_docs[i][0]; i++) {
-        if (strcmp(tok->str_val, builtin_docs[i][0]) == 0) {
-            hover_text = builtin_docs[i][1];
-            break;
+    if (!dotted)
+        for (int i = 0; builtin_docs[i][0]; i++) {
+            if (strcmp(tok->str_val, builtin_docs[i][0]) == 0) {
+                hover_text = builtin_docs[i][1];
+                break;
+            }
         }
-    }
 
     /* Check keywords */
-    if (!hover_text) {
+    if (!hover_text && !dotted) {
         for (int i = 0; keyword_docs[i][0]; i++) {
             if (strcmp(tok->str_val, keyword_docs[i][0]) == 0) {
                 hover_text = keyword_docs[i][1];
@@ -1090,7 +1037,7 @@ static void handle_hover(int id, const char *params) {
     }
 
     /* Check document symbols */
-    if (!hover_text) {
+    if (!hover_text && !dotted) {
         for (int i = 0; i < doc->symbol_count; i++) {
             Symbol *s = &doc->symbols[i];
             if (strcmp(tok->str_val, s->name) == 0) {
@@ -1127,8 +1074,7 @@ static void handle_hover(int id, const char *params) {
      * qualifier is not an imported module is left alone); a bare name
      * matches any imported module's function in table order. */
     if (!hover_text) {
-        ptrdiff_t ti = tok - doc->tokens.tokens;
-        int dotted = (ti >= 1 && doc->tokens.tokens[ti - 1].type == TOK_DOT);
+        /* ti/dotted computed above, before the builtin lookup. */
         const char *dot_module = NULL;
         if (dotted && ti >= 2) {
             Token *t2 = &doc->tokens.tokens[ti - 2];

src/eigenscript.h

@@ -1304,6 +1304,10 @@ void   handle_release(int id);
 /* ---- EigenStore embedded database ---- */
 void register_store_builtins(Env *env);
 
+/* ---- gfx extension registrar (ext_gfx.c; TU only compiled when
+ * EIGENSCRIPT_EXT_GFX — call sites keep the #if, matching http/db). ---- */
+void register_gfx_builtins(Env *env);
+
 /* ---- Tape-stepper (#418; step.c, CLI-only) ----
  * Interactive debugger over a recorded trace tape: `--step <tape> [src]`.
  * Returns the process exit code (3 = version refusal, the replay rule). */

src/eigs_embed.c

@@ -21,8 +21,7 @@ int eigs_state_init_runtime(EigsState *st) {
     if (g_global_env) return 0;
     Env *global = env_new(NULL);
     if (!global) return -1;
-    register_builtins(global);
-    register_store_builtins(global);
+    register_builtins(global);   /* one seam: store/gfx ride inside (#742) */
     g_global_env = global;
     return 0;
 }

src/embed_smoke.c

@@ -88,6 +88,25 @@ int main(void) {
     CHECK(eigs_value_as_num(r) == 35.0, "5 * 7 == 35");
     eigs_value_release(r);
 
+    /* --- Composition seam (#742): the embed API builds the global env
+     * through register_builtins alone, so every compiled-in extension is
+     * present here, not just through the CLI. store is always compiled;
+     * gfx rides when built with EIGENSCRIPT_EXT_GFX (pre-#742 only main.c
+     * registered gfx, so a gfx build used through this API had no gfx
+     * builtins at all — `make embed-smoke-gfx` pins that leg). ---------- */
+    r = eigs_eval_string("type of store_open");
+    CHECK(r != NULL && eigs_value_type(r) == EIGS_TYPE_STR &&
+              strcmp(eigs_value_as_string(r), "builtin") == 0,
+          "store_open composed via the embed seam");
+    eigs_value_release(r);
+#if EIGENSCRIPT_EXT_GFX
+    r = eigs_eval_string("type of gfx_open");
+    CHECK(r != NULL && eigs_value_type(r) == EIGS_TYPE_STR &&
+              strcmp(eigs_value_as_string(r), "builtin") == 0,
+          "gfx_open composed via the embed seam (#742)");
+    eigs_value_release(r);
+#endif
+
     /* --- Read it back through the globals API. ----------------------- */
     EigsValue *y = eigs_get_global("y");
     CHECK(y != NULL, "get_global y");

src/ext_http.c

@@ -1405,8 +1405,7 @@ static void *http_conn_thread(void *arg) {
     }
 
     Env *global = env_new(NULL);
-    register_builtins(global);
-    register_store_builtins(global);
+    register_builtins(global);   /* one seam: store/gfx ride inside (#742) */
     g_global_env = global;
 
     /* register_http_builtins allocated a scratch Server on the worker

src/lint.c

@@ -157,8 +157,7 @@ static void builtin_name_env_free(void) {
 static int is_builtin_name(const char *name) {
     if (!g_builtin_name_env) {
         Env *e = env_new(NULL);
-        register_builtins(e);
-        register_store_builtins(e);
+        register_builtins(e);   /* store/gfx-when-built ride inside (#742) */
 #define X(nm, fn) if (!env_get(e, #nm)) env_set_local_owned(e, #nm, make_null());
         EIGS_GFX_BUILTINS(X)
         EIGS_HTTP_BUILTINS(X)
@@ -1142,8 +1141,7 @@ static int api_is_ext_name(const char *nm) {
 int eigs_api_dump(FILE *out, int json) {
     /* Core registry on a scratch env (#459: never a hand list). */
     Env *core = env_new(NULL);
-    register_builtins(core);
-    register_store_builtins(core);
+    register_builtins(core);   /* store/gfx-when-built ride inside (#742) */
 
     /* Lib scan over the resolver's candidate dirs. */
     ApiLibFn *fns = NULL;
@@ -3512,8 +3510,7 @@ static void check_undefined_names(ASTNode *ast, const char *path,
     e.bind = env_new(NULL);
     e.module_scope = env_new(e.bind);
     e.scope = e.module_scope;
-    register_builtins(e.bind);
-    register_store_builtins(e.bind);
+    register_builtins(e.bind);   /* store/gfx-when-built ride inside (#742) */
     /* Extension builtins bind by NAME regardless of this binary's build
      * flags (ext_names.h, the same lists their registrars expand): the lint
      * describes the language surface, not the build — a consumer linting

src/main.c

@@ -7,9 +7,6 @@
 #include "vm.h"
 #include "trace.h"
 #include "repl.h"
-#if EIGENSCRIPT_EXT_GFX
-void register_gfx_builtins(Env *env);
-#endif
 
 #ifndef EIGENSCRIPT_VERSION
 #define EIGENSCRIPT_VERSION "dev"
@@ -271,14 +268,9 @@ int main(int argc, char **argv) {
         eigenscript_set_args(argc, argv);
 
         Env *global = env_new(NULL);
-        register_builtins(global);
+        register_builtins(global);   /* one seam: store/gfx ride inside (#742) */
         g_global_env = global;
 
-#if EIGENSCRIPT_EXT_GFX
-        register_gfx_builtins(global);
-#endif
-        register_store_builtins(global);
-
         eigenscript_repl(global);
         /* #739: take the exit code BEFORE teardown. It is a bridge macro now
          * (state latch, reached through eigs_current), so eigs_thread_detach
@@ -327,12 +319,7 @@ int main(int argc, char **argv) {
 
     Env *global = env_new(NULL);
     register_builtins(global);
-    g_global_env = global;
-
-#if EIGENSCRIPT_EXT_GFX
-    register_gfx_builtins(global);
-#endif
-    register_store_builtins(global);
+    g_global_env = global;   /* register_builtins above composed store/gfx too (#742) */
 
     g_parse_errors = 0;
     TokenList tl = tokenize(source);