Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -214,6 +214,7 @@ questions — ask `lmc`.** The agent then runs this loop:
lmc init --auto --path .
walk tree, count file extensions → dominant language
→ write lumos.yml {language, codebase_hash = sha1(abspath)[:16]}
optional, by hand: exclude: [<dir>, ...] → project-specific dirs to skip

lmc build --path .
gateway: graph.build_index() → in-memory Index (methods + call edges)
Expand Down Expand Up @@ -353,6 +354,17 @@ Eigenes Image `lmc-joern:latest` aus `docker/Dockerfile` (basiert auf
pysrc2cpg, … je nach Sprache — keine Excludes mehr im generic `joern-parse`);
`lmc query` lädt sie per `importCpg` und führt CPGQL aus.

Ausgeschlossen werden Dependency- und Build-Ordner (`vendor`, `node_modules`,
`.worktrees`, …). Was nur in *einem* Projekt stört — ein mitgeschleppter
Legacy-Baum etwa —, kommt in die `lumos.yml` und gilt dann für beide Engines:

```yaml
language: php
codebase_hash: d870a6000cbcd0ae
exclude:
- legacy_app
```

Manuell bauen (optional, vorab):
```bash
docker build -t lmc-joern:latest docker/
Expand Down
18 changes: 17 additions & 1 deletion lmc/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@
IGNORE_DIRS = {
".git", "node_modules", "vendor", ".venv", "venv", "dist", "build",
"__pycache__", ".idea", ".vscode", "target", "bower_components",
"storage", "var", "cache",
"storage", "var", "cache", ".worktrees", ".phpstan",
}

CONFIG_NAME = "lumos.yml"
Expand Down Expand Up @@ -90,6 +90,22 @@ def load_config(path) -> dict:
return yaml.safe_load(cfg.read_text(encoding="utf-8")) or {}


def excludes_from_config(path) -> set:
"""Projekteigene Ausschluesse aus dem `exclude:`-Key der lumos.yml.

Fuer Verzeichnisse, die nur in *diesem* Projekt stoeren und darum nichts in
IGNORE_DIRS/DEFAULT_EXCLUDES zu suchen haben — etwa ein mitgeschleppter
Legacy-Baum. Erwartet Top-Level-Namen relativ zur Worktree-Wurzel:

exclude:
- legacy_app

Liefert eine leere Menge, wenn es keine lumos.yml oder keinen Key gibt.
"""
raw = load_config(path).get("exclude") or []
return {str(d).strip("/") for d in raw if str(d).strip("/")}


def save_config(path, language: str, extra: dict | None = None) -> Path:
"""Schreibt lumos.yml mit Sprache + optionalem extra (z.B. codebase_hash)."""
cfg_path = Path(path).resolve() / CONFIG_NAME
Expand Down
22 changes: 16 additions & 6 deletions lmc/joern.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,12 @@
import re
import subprocess
import time
from typing import Optional
from typing import Iterable, Optional

import httpx

from lmc.config import excludes_from_config

JOERN_HOST = "127.0.0.1"
JOERN_PORT = 8085
IMAGE = "lmc-joern:latest"
Expand All @@ -31,6 +33,8 @@
DEFAULT_EXCLUDES = (
"vendor", "node_modules", ".venv", "venv", "__pycache__",
"storage", "public", "dist", "build", "target", ".git",
".worktrees", # git worktrees: ganze Zweit-Checkouts, sonst kommt jeder Treffer n-fach
".phpstan", # PHPStan-Cache: resultCache.php wird zweistellig MB gross -> OOM beim Einlesen
)

# ponytail: joern-parse kennt kein --exclude, und was nach --frontend-args kommt,
Expand Down Expand Up @@ -137,11 +141,13 @@ def cpg_path_in_container(codebase_hash: str) -> str:
return f"{CPG_DIR_IN_CONTAINER}/{codebase_hash}.bin"


def parse_command(worktree: str, out: str, language: str | None = None) -> list[str]:
def parse_command(worktree: str, out: str, language: str | None = None,
extra_excludes: Iterable[str] = ()) -> list[str]:
"""Baut das docker-Kommando fuer den CPG-Lauf.

Mit bekannter Sprache das Frontend direkt (dann greifen die Excludes),
sonst unveraendert `joern-parse` ueber den ganzen Baum.
sonst unveraendert `joern-parse` ueber den ganzen Baum. `extra_excludes`
kommt aus dem `exclude:`-Key der lumos.yml und ergaenzt DEFAULT_EXCLUDES.
"""
base = [
"docker", "run", "--rm",
Expand All @@ -153,15 +159,19 @@ def parse_command(worktree: str, out: str, language: str | None = None) -> list[
if frontend is None:
return base + ["joern-parse", "/code", "-o", out]
excludes = []
for d in DEFAULT_EXCLUDES:
for d in dict.fromkeys((*DEFAULT_EXCLUDES, *extra_excludes)):
excludes += ["--exclude", f"/code/{d}"]
return base + [frontend, "/code", *excludes, "-o", out]


def joern_parse(worktree: str, codebase_hash: str, language: str | None = None) -> dict:
"""Baut <hash>.bin im Volume aus dem Worktree im Container."""
"""Baut <hash>.bin im Volume aus dem Worktree im Container.

Die lumos.yml wird im gemounteten Baum gesucht; bei `--scope` zeigt
`worktree` auf den Teilbaum, dann greifen nur DEFAULT_EXCLUDES.
"""
out = cpg_path_in_container(codebase_hash)
cmd = parse_command(worktree, out, language)
cmd = parse_command(worktree, out, language, excludes_from_config(worktree))
try:
r = subprocess.run(cmd, capture_output=True, text=True, timeout=600)
except FileNotFoundError:
Expand Down
9 changes: 7 additions & 2 deletions lmc/server/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

from tree_sitter_language_pack import get_parser

from lmc.config import excludes_from_config

#Unsere Sprachen -> tree-sitter-Sprachname.
TS_LANG = {
"php": "php", "python": "python", "javascript": "javascript",
Expand Down Expand Up @@ -292,11 +294,14 @@ def build_index(codebase_hash: str, language: str, root_path) -> Index:
root = Path(root_path).resolve()
idx = Index(codebase_hash=codebase_hash, language=language)
exts = _exts_for(language)
# Projekteigene Ausschluesse aus der lumos.yml im Baum; bei --scope liegt dort
# keine, dann bleibt es bei _IGNORE.
ignore = _IGNORE | excludes_from_config(root)

for p in root.rglob("*"):
if not p.is_file() or p.suffix.lower() not in exts:
continue
if any(part in _IGNORE for part in p.relative_to(root).parts[:-1]):
if any(part in ignore for part in p.relative_to(root).parts[:-1]):
continue
try:
src_str = p.read_text(encoding="utf-8", errors="replace") # parse braucht str (tslp 1.11.0)
Expand Down Expand Up @@ -337,7 +342,7 @@ def _walk(node, src, file, idx, def_stack):
_IGNORE = {
".git", "node_modules", "vendor", ".venv", "venv", "dist", "build",
"__pycache__", ".idea", ".vscode", "target", "bower_components",
"storage", "var", "cache", "__pycache__",
"storage", "var", "cache", "__pycache__", ".worktrees", ".phpstan",
}

_LANG_EXTS = {
Expand Down
97 changes: 97 additions & 0 deletions tests/test_config_excludes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
"""Projekteigene Ausschluesse aus dem `exclude:`-Key der lumos.yml.

Motivation 2026-08-14: ein Laravel-Repo schleppte einen Legacy-Baum mit
1,5-MB-Font-Arrays mit, an dem php2cpg mit ``java.lang.OutOfMemoryError`` starb.
So ein Ordnername ist projektspezifisch und hat in DEFAULT_EXCLUDES nichts zu
suchen — dafuer gibt es jetzt ``exclude:`` in der lumos.yml.

Generische Dependency-/Cache-Ordner bleiben dagegen in den Default-Listen; der
Test haelt beide Ebenen auseinander.
"""
import os
import sys
import tempfile
from pathlib import Path

# Repo-Root zum Pfad hinzufuegen (laeuft ohne Install).
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

from lmc.config import excludes_from_config, save_config # noqa: E402
from lmc.joern import DEFAULT_EXCLUDES, parse_command # noqa: E402

OUT = "/cpgs/deadbeef.bin"


def _worktree(tmp: str, exclude=None) -> str:
"""Minimaler Worktree mit lumos.yml — genau das, was `lmc init` schreibt."""
extra = {"codebase_hash": "deadbeef"}
if exclude is not None:
extra["exclude"] = exclude
save_config(tmp, "php", extra=extra)
return tmp


def test_no_config_means_no_extra_excludes():
"""Ohne lumos.yml darf nichts explodieren — leere Menge, kein Fehler."""
with tempfile.TemporaryDirectory() as d:
assert excludes_from_config(d) == set()


def test_missing_key_means_no_extra_excludes():
"""lumos.yml ohne `exclude:` ist der Normalfall (so schreibt `lmc init` sie)."""
with tempfile.TemporaryDirectory() as d:
assert excludes_from_config(_worktree(d)) == set()


def test_exclude_key_is_read():
"""Der eigentliche Zweck: Namen aus der Config kommen an, Slashes normalisiert."""
with tempfile.TemporaryDirectory() as d:
got = excludes_from_config(_worktree(d, ["legacy", "third_party/"]))
assert got == {"legacy", "third_party"}, got


def test_config_excludes_reach_the_frontend_command():
"""Ohne diesen Schritt liest die Config zwar, der CPG-Bau ignoriert sie aber."""
cmd = parse_command("/w", OUT, "php", extra_excludes={"legacy"})
assert "/code/legacy" in cmd, cmd
assert "/code/vendor" in cmd, "Defaults duerfen nicht verdraengt werden"


def test_config_excludes_are_not_duplicated():
"""`vendor` doppelt (Default + Config) wuerde php2cpg zwei gleiche Flags geben."""
cmd = parse_command("/w", OUT, "php", extra_excludes={"vendor"})
assert cmd.count("/code/vendor") == 1
assert cmd.count("--exclude") == len(DEFAULT_EXCLUDES)


def test_index_skips_configured_dir():
"""Der Beweis am Ergebnis: die Datei taucht nicht im Index auf."""
from lmc.server.graph import build_index

with tempfile.TemporaryDirectory() as d:
(Path(d) / "app").mkdir()
(Path(d) / "app" / "Keep.php").write_text("<?php\nfunction keep_me() {}\n")
(Path(d) / "legacy").mkdir()
(Path(d) / "legacy" / "Drop.php").write_text("<?php\nfunction drop_me() {}\n")

_worktree(d) # noch ohne exclude
names = {m.name for m in build_index("deadbeef", "php", d).methods}
assert names == {"keep_me", "drop_me"}, f"Baseline stimmt nicht: {names}"

_worktree(d, ["legacy"])
names = {m.name for m in build_index("deadbeef", "php", d).methods}
assert names == {"keep_me"}, f"legacy/ haette rausfallen muessen: {names}"


if __name__ == "__main__":
for fn in (
test_no_config_means_no_extra_excludes,
test_missing_key_means_no_extra_excludes,
test_exclude_key_is_read,
test_config_excludes_reach_the_frontend_command,
test_config_excludes_are_not_duplicated,
test_index_skips_configured_dir,
):
fn()
print(f"ok: {fn.__name__}")
print("all lumos.yml exclude tests passed")