mganter/knowledge-base-framework
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: extract kb_framework package, build graph in memory

Browse Source 
Scripts become thin CLIs over kb_framework (graph, query, query_tools,
vault, instructions) so the hosted read-only connector shares one
implementation with the stdio server. The MCP server builds the graph
from Markdown on demand and invalidates it on writes; refresh_graph
rebuilds in memory and reports validation problems. Pre-commit hook
validates only; index/graph.json is no longer read, written or staged
(generate-graph.py --write remains for operators). Golden fixtures pin
graph and tool payloads to the previous output. Agent and skill docs
updated to match.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
TotenKnopf
2026-10-06 20:40:13 +02:00
co-authored by Claude Fable 5.1
parent 4fec637599
commit 96780317a7
37 changed files with 1858 additions and 899 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/generate-graph.py
+54 -466
View File
@@ -1,439 +1,51 @@
#!/usr/bin/env python3
"""Build `index/graph.json` from the vault's Markdown, and validate it.
"""Validate the vault graph, and optionally write it to `index/graph.json`.
The graph is the derived navigation layer agents query instead of walking the
tree: one node per note (frontmatter plus link counts) and one edge per typed
markdown link. `--validate` is the gate that keeps it trustworthy — it fails on
links that resolve to nothing, on edge relationship types or note `type` values
outside their controlled vocabularies, and on legacy `[[wiki links]]`.
Thin CLI over `kb_framework.graph`: the graph is built in memory from the
vault's Markdown. `--validate` is the gate the pre-commit hook applies — it
fails on links that resolve to nothing, on edge relationship types or note
`type` values outside their controlled vocabularies, and on legacy
`[[wiki links]]`. `--write` is for operators who want the file; nothing else
writes it, and the MCP servers never read it.
"""
import argparse
import json
import re
import sys
from collections import defaultdict
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
from typing import Dict, List, Tuple
from typing import Dict, List
from vault_root import parse_vault_args
# Obsidian-style wiki link: [[target#anchor|alias]]
WIKI_LINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|([^\]]+))?\]\]")
# Standard markdown link: [display](target); (?<!!) excludes ![img](...)
MD_LINK_RE = re.compile(r"(?<!!)\[([^\]]*)\]\(([^)]+)\)")
# Trailing "— gloss" (em-dash / en-dash / hyphen) used as an annotation fallback
GLOSS_RE = re.compile(r"^\s*[—–-]\s+(\S.*)$")
FENCED_CODE_RE = re.compile(r"```.*?```", re.DOTALL)
INLINE_CODE_RE = re.compile(r"`[^`\n]+`")
# Markdown targets that are external (not vault edges, not unresolved-link errors)
EXTERNAL_TARGET_RE = re.compile(r"^(?:[a-z][a-z0-9+.\-]*://|mailto:|tel:|#)", re.IGNORECASE)
# Strict controlled vocabulary for edge relationship types. A link with no brace
# (or no "type") defaults to "related"; --validate fails on any type not in this set.
RELATION_TYPES = {
"related",
"supersedes",
"superseded-by",
"depends-on",
"part-of",
"parent",
"derived-from",
"see-also",
"source",
}
# Strict controlled vocabulary for a note's frontmatter `type`. A note without
# one is a "note"; --validate fails on any value outside this set, which is what
# keeps the facet meaningful as the vault grows.
NOTE_TYPES = {
"hub",
"project",
"area",
"guideline",
"policy",
"feature",
"roadmap",
"decision",
"note",
"meta",
"reference",
}
@dataclass
class LinkRef:
target: str # raw target text (resolution key: id / rel-path / title / stem, or a URL)
anchor: str # display / alias text -> edge anchor_text
kind: str # "wiki_link" | "markdown_link"
rel_type: str # semantic relationship type (default "related")
annotation: str # per-edge "what to expect there" note
lineno: int # 1-based line number in the note body
raw: str # the raw link text as written (for the inventory script)
is_external: bool # markdown link whose target is a URL/anchor -> no edge
@dataclass
class Note:
path: Path
rel_path: str
stem: str
meta: Dict[str, object]
title: str
note_id: str
links: List[LinkRef]
@dataclass(frozen=True)
class UnresolvedLink:
from_id: str
from_path: str
target: str
@dataclass(frozen=True)
class WikiLinkUse:
from_id: str
from_path: str
target: str
lineno: int
raw: str
def parse_frontmatter(text: str) -> Tuple[Dict[str, object], str]:
if not text.startswith("---\n"):
return {}, text
end = text.find("\n---\n", 4)
if end == -1:
return {}, text
fm_block = text[4:end]
body = text[end + 5 :]
meta: Dict[str, object] = {}
for raw in fm_block.splitlines():
line = raw.strip()
if not line or line.startswith("#") or ":" not in line:
continue
key, value = line.split(":", 1)
key = key.strip()
value = value.strip()
if value.startswith("[") and value.endswith("]"):
inner = value[1:-1].strip()
if inner:
meta[key] = [v.strip().strip('"').strip("'") for v in inner.split(",")]
else:
meta[key] = []
else:
meta[key] = value.strip('"').strip("'")
return meta, body
def derive_title(meta: Dict[str, object], body: str, stem: str) -> str:
if isinstance(meta.get("title"), str) and meta["title"].strip():
return str(meta["title"]).strip()
for line in body.splitlines():
if line.startswith("# "):
return line[2:].strip()
return stem.replace("-", " ").strip().title()
def mask_code(body: str) -> str:
"""Blank out code so links inside code samples are ignored, preserving line
boundaries so line-based parsing (gloss fallback, line numbers) stays aligned."""
body = FENCED_CODE_RE.sub(lambda m: "\n" * m.group(0).count("\n"), body)
body = INLINE_CODE_RE.sub(lambda m: " " * len(m.group(0)), body)
return body
def scan_brace(text: str, start: int) -> Tuple[object, int]:
"""Given text[start] == '{', scan to the matching '}' respecting JSON string
quoting, and json.loads the block. Returns (obj_or_None, end_index_after_brace).
On an unterminated or malformed brace, returns (None, start) so the caller
treats it as no brace."""
depth = 0
in_str = False
esc = False
i = start
while i < len(text):
c = text[i]
if in_str:
if esc:
esc = False
elif c == "\\":
esc = True
elif c == '"':
in_str = False
elif c == '"':
in_str = True
elif c == "{":
depth += 1
elif c == "}":
depth -= 1
if depth == 0:
raw = text[start : i + 1]
try:
return json.loads(raw), i + 1
except (json.JSONDecodeError, ValueError):
return None, start
i += 1
return None, start
def clean_md_target(raw: str) -> str:
"""Normalize a markdown link target: drop a trailing ``"title"`` and <> wrappers."""
raw = raw.strip()
if raw.startswith("<") and raw.endswith(">"):
raw = raw[1:-1].strip()
# `[text](target "title")` -> target
for sep in (' "', " '"):
if sep in raw:
raw = raw.split(sep, 1)[0].strip()
return raw
def parse_links(body: str) -> List[LinkRef]:
masked = mask_code(body)
refs: List[LinkRef] = []
for lineno, line in enumerate(masked.splitlines(), start=1):
# Collect wiki + markdown link matches on this line, in order.
raw_matches: List[Tuple[int, int, str, str, str, str]] = []
for m in WIKI_LINK_RE.finditer(line):
target = m.group(1).strip()
alias = (m.group(2) or target).strip()
if target:
raw_matches.append(
(m.start(), m.end(), "wiki_link", target, alias, m.group(0))
)
for m in MD_LINK_RE.finditer(line):
target = clean_md_target(m.group(2))
display = m.group(1).strip()
if target:
raw_matches.append(
(
m.start(),
m.end(),
"markdown_link",
target,
display or target,
m.group(0),
)
)
raw_matches.sort(key=lambda x: x[0])
line_refs: List[Tuple[LinkRef, int]] = [] # (ref, end-index-after-brace)
for start, end, kind, target, anchor, raw in raw_matches:
rel_type, annotation = "related", ""
brace_end = end
if end < len(line) and line[end] == "{":
obj, be = scan_brace(line, end)
if isinstance(obj, dict):
brace_end = be
t = obj.get("type")
if isinstance(t, str) and t.strip():
rel_type = t.strip()
info = obj.get("link-info")
if isinstance(info, str) and info.strip():
annotation = info.strip()
is_external = kind == "markdown_link" and bool(
EXTERNAL_TARGET_RE.match(target)
)
ref = LinkRef(
target=target,
anchor=anchor,
kind=kind,
rel_type=rel_type,
annotation=annotation,
lineno=lineno,
raw=raw,
is_external=is_external,
)
line_refs.append((ref, brace_end))
# Annotation fallback: a trailing "— gloss" applies only when the link is
# the sole link on its line (so we don't mis-assign a shared gloss).
if len(line_refs) == 1:
ref, brace_end = line_refs[0]
if not ref.annotation:
gloss = GLOSS_RE.match(line[brace_end:])
if gloss:
ref.annotation = gloss.group(1).strip()
refs.extend(ref for ref, _ in line_refs)
return refs
def norm(s: str) -> str:
return s.strip().lower()
def build_notes(root: Path) -> List[Note]:
notes: List[Note] = []
for p in sorted(root.rglob("*.md")):
rel = p.relative_to(root)
if rel.parts and rel.parts[0] in {"index", "scripts", ".obsidian"}:
continue
# `raw/` holds untouched ingested originals (external READMEs, specs, …),
# not vault notes — skip them so their file-relative markdown links are not
# treated as (broken) graph edges.
if "raw" in rel.parts:
continue
text = p.read_text(encoding="utf-8")
meta, body = parse_frontmatter(text)
title = derive_title(meta, body, p.stem)
note_id = str(meta.get("id") or rel.with_suffix("")).strip()
links = parse_links(body)
# parse_links numbers lines within `body`; shift to absolute file line
# numbers by the number of lines the frontmatter block occupied.
line_offset = text.count("\n", 0, len(text) - len(body))
if line_offset:
for ref in links:
ref.lineno += line_offset
notes.append(
Note(
path=p,
rel_path=rel.as_posix(),
stem=p.stem,
meta=meta,
title=title,
note_id=note_id,
links=links,
)
)
return notes
def build_alias_map(notes: List[Note]) -> Dict[str, str]:
alias_map: Dict[str, str] = {}
stem_buckets: Dict[str, List[str]] = defaultdict(list)
for n in notes:
stem_buckets[norm(n.stem)].append(n.note_id)
for n in notes:
alias_map[norm(n.note_id)] = n.note_id
alias_map[norm(n.rel_path)] = n.note_id
alias_map[norm(Path(n.rel_path).with_suffix("").as_posix())] = n.note_id
alias_map[norm(n.title)] = n.note_id
for stem, ids in stem_buckets.items():
if len(ids) == 1:
alias_map[stem] = ids[0]
return alias_map
def generate_graph(
root: Path, existing_generated_at: str = ""
) -> Tuple[Dict[str, object], List[UnresolvedLink], List[WikiLinkUse]]:
notes = build_notes(root)
alias_map = build_alias_map(notes)
edges_set = set()
outbound: Dict[str, List[str]] = defaultdict(list)
inbound_count: Dict[str, int] = defaultdict(int)
unresolved_set = set()
wiki_uses_set = set()
for n in notes:
for ref in n.links:
# Legacy [[wiki links]] are still parsed (so we can flag them) but are
# no longer an accepted link form — --validate rejects any that remain.
if ref.kind == "wiki_link":
wiki_uses_set.add(
WikiLinkUse(
from_id=n.note_id,
from_path=n.rel_path,
target=ref.target,
lineno=ref.lineno,
raw=ref.raw,
)
)
if ref.is_external:
continue
target = alias_map.get(norm(ref.target))
if not target:
unresolved_set.add(
UnresolvedLink(
from_id=n.note_id,
from_path=n.rel_path,
target=ref.target,
)
)
continue
edge = (n.note_id, target, ref.kind, ref.rel_type, ref.annotation, ref.anchor)
if edge in edges_set:
continue
edges_set.add(edge)
outbound[n.note_id].append(target)
inbound_count[target] += 1
nodes = []
for n in sorted(notes, key=lambda x: x.note_id):
tags = n.meta.get("tags", [])
if not isinstance(tags, list):
tags = []
nodes.append(
{
"id": n.note_id,
"title": n.title,
"path": n.rel_path,
"repo": n.meta.get("repo", "global"),
"type": n.meta.get("type", "note"),
"status": n.meta.get("status", "active"),
"tags": tags,
"last_reviewed": n.meta.get("last_reviewed", ""),
"outbound_links": sorted(set(outbound.get(n.note_id, []))),
"inbound_count": inbound_count.get(n.note_id, 0),
}
)
edges = [
{
"from": src,
"to": dst,
"kind": kind,
"type": rel_type,
"anchor_text": anchor,
"annotation": annotation,
"weight": 1.0,
}
for src, dst, kind, rel_type, annotation, anchor in sorted(edges_set)
]
graph = {
"version": 1,
"generated_at": existing_generated_at
if existing_generated_at
else datetime.now(timezone.utc).isoformat(),
"nodes": nodes,
"edges": edges,
}
unresolved = sorted(unresolved_set, key=lambda x: (x.from_path, x.target))
wiki_uses = sorted(wiki_uses_set, key=lambda x: (x.from_path, x.lineno, x.raw))
return graph, unresolved, wiki_uses
from kb_framework.graph import ( # noqa: F401 (re-exported for list-wiki-links.py and tests)
NOTE_TYPES,
RELATION_TYPES,
GraphBuild,
LinkRef,
Note,
UnresolvedLink,
WikiLinkUse,
build_alias_map,
build_graph,
build_notes,
find_invalid_node_types,
find_invalid_types,
generate_graph,
norm,
parse_frontmatter,
parse_links,
)
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Generate and validate Obsidian graph JSON"
description="Validate the vault graph; optionally write index/graph.json"
)
parser.add_argument(
"--validate",
action="store_true",
help="validate the graph: unresolved targets, edge relationship types, "
"note types, "
"and legacy [[wiki links]] (all rejected)",
"note types, and legacy [[wiki links]] (all rejected)",
)
parser.add_argument(
"--write",
@@ -447,7 +59,6 @@ def print_unresolved(unresolved: List[UnresolvedLink]) -> None:
if not unresolved:
print("Validation passed: no unresolved links")
return
print(f"Validation failed: {len(unresolved)} unresolved link(s)")
for link in unresolved:
print(f"- {link.from_path}: [[{link.target}]]")
@@ -457,92 +68,69 @@ def print_wiki_links(wiki_uses: List[WikiLinkUse]) -> None:
if not wiki_uses:
print("Validation passed: no legacy [[wiki links]]")
return
print(f"Validation failed: {len(wiki_uses)} legacy [[wiki link(s)]]")
print(" use typed markdown links; find/convert them with scripts/list-wiki-links.py")
for w in wiki_uses:
print(f"- {w.from_path}:{w.lineno}: {w.raw}")
def find_invalid_types(graph: Dict[str, object]) -> List[Dict[str, object]]:
return [
e
for e in graph.get("edges", []) # type: ignore[union-attr]
if e.get("type") not in RELATION_TYPES
]
def print_invalid_types(invalid: List[Dict[str, object]]) -> None:
if not invalid:
print("Validation passed: all edge types are in the controlled vocabulary")
return
print(f"Validation failed: {len(invalid)} edge(s) with unknown relationship type")
print(f" allowed: {', '.join(sorted(RELATION_TYPES))}")
for e in invalid:
print(f"- {e.get('from')} -> {e.get('to')}: type={e.get('type')!r}")
def find_invalid_node_types(graph: Dict[str, object]) -> List[Dict[str, object]]:
return [
n
for n in graph.get("nodes", []) # type: ignore[union-attr]
if n.get("type") not in NOTE_TYPES
]
def print_invalid_node_types(invalid: List[Dict[str, object]]) -> None:
if not invalid:
print("Validation passed: all node types are in the controlled vocabulary")
return
print(f"Validation failed: {len(invalid)} note(s) with unknown type")
print(f" allowed: {', '.join(sorted(NOTE_TYPES))}")
for n in invalid:
print(f"- {n.get('id')} ({n.get('path')}): type={n.get('type')!r}")
def _existing_generated_at(output: Path, candidate: GraphBuild) -> str:
"""Keep the old timestamp when the written graph would be unchanged."""
if not output.exists():
return ""
try:
existing = json.loads(output.read_text(encoding="utf-8"))
except (json.JSONDecodeError, OSError, TypeError, ValueError):
return ""
if (
candidate.graph["nodes"] == existing.get("nodes", [])
and candidate.graph["edges"] == existing.get("edges", [])
):
return str(existing.get("generated_at", ""))
return ""
def main() -> None:
args = parse_args()
root = args.vault_root
output = root / "index" / "graph.json"
should_write = args.write or not args.validate
build = build_graph(root)
if should_write:
stamp = _existing_generated_at(output, build)
if stamp:
build = build_graph(root, existing_generated_at=stamp)
output.parent.mkdir(parents=True, exist_ok=True)
existing_generated_at = ""
if output.exists():
try:
existing = json.loads(output.read_text(encoding="utf-8"))
existing_nodes = existing.get("nodes", [])
existing_edges = existing.get("edges", [])
candidate_graph, unresolved, wiki_uses = generate_graph(root)
if (
candidate_graph["nodes"] == existing_nodes
and candidate_graph["edges"] == existing_edges
):
existing_generated_at = str(existing.get("generated_at", ""))
graph, unresolved, wiki_uses = generate_graph(
root, existing_generated_at=existing_generated_at
)
except (json.JSONDecodeError, OSError, TypeError, ValueError):
graph, unresolved, wiki_uses = generate_graph(root)
else:
graph, unresolved, wiki_uses = generate_graph(root)
if should_write:
output.write_text(json.dumps(graph, indent=2) + "\n", encoding="utf-8")
output.write_text(json.dumps(build.graph, indent=2) + "\n", encoding="utf-8")
print(f"Wrote {output}")
if args.validate:
print_unresolved(unresolved)
invalid = find_invalid_types(graph)
print_invalid_types(invalid)
invalid_nodes = find_invalid_node_types(graph)
print_invalid_node_types(invalid_nodes)
print_wiki_links(wiki_uses)
if unresolved or invalid or invalid_nodes or wiki_uses:
print_unresolved(build.unresolved)
print_invalid_types(build.invalid_edge_types)
print_invalid_node_types(build.invalid_node_types)
print_wiki_links(build.wiki_uses)
if not build.ok:
sys.exit(1)