commit 638cbe6b3947c37c5348f25b8fd6a882ce804f2a Author: Manuel Ganter Date: Fri Sep 11 09:40:02 2026 +0200 stripped down to skeleton diff --git a/.claude/agents/the-knowing-one.md b/.claude/agents/the-knowing-one.md new file mode 120000 index 0000000..d01f6a1 --- /dev/null +++ b/.claude/agents/the-knowing-one.md @@ -0,0 +1 @@ +../../agents/the-knowing-one.md \ No newline at end of file diff --git a/.claude/agents/the-scribe.md b/.claude/agents/the-scribe.md new file mode 120000 index 0000000..2303958 --- /dev/null +++ b/.claude/agents/the-scribe.md @@ -0,0 +1 @@ +../../agents/the-scribe.md \ No newline at end of file diff --git a/.claude/skills/knowledge-base-ingest-resource b/.claude/skills/knowledge-base-ingest-resource new file mode 120000 index 0000000..0e5a34f --- /dev/null +++ b/.claude/skills/knowledge-base-ingest-resource @@ -0,0 +1 @@ +../../skills/knowledge-base-ingest-resource \ No newline at end of file diff --git a/.claude/skills/knowledge-base-read b/.claude/skills/knowledge-base-read new file mode 120000 index 0000000..4998cc0 --- /dev/null +++ b/.claude/skills/knowledge-base-read @@ -0,0 +1 @@ +../../skills/knowledge-base-read \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..dd7f95f --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +*.pyc +.claude/settings.local.json diff --git a/01_projects/.gitkeep b/01_projects/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/01_projects/index.md b/01_projects/index.md new file mode 100644 index 0000000..9d1acfe --- /dev/null +++ b/01_projects/index.md @@ -0,0 +1,20 @@ +--- +id: project/index +title: Projects Hub +type: hub +status: active +tags: [project, hub] +last_reviewed: 2026-09-11 +--- + +# Projects Hub + +Active efforts with a goal and an end, one folder each under `01_projects/`. +A project finishes — when it does, move it to `04_archives/`. + +## Projects +- // no document, yet + +## Related +- [Areas Hub](area/index){"type":"related","link-info":"Links to the sibling Areas hub"} +- [Resources Hub](resource/index){"type":"related","link-info":"Links to the sibling Resources hub"} diff --git a/02_areas/.gitkeep b/02_areas/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/02_areas/index.md b/02_areas/index.md new file mode 100644 index 0000000..60ca6db --- /dev/null +++ b/02_areas/index.md @@ -0,0 +1,20 @@ +--- +id: area/index +title: Areas Hub +type: hub +status: active +tags: [area, hub] +last_reviewed: 2026-09-11 +--- + +# Areas Hub + +Ongoing responsibilities & standards, one folder each under `02_areas/`. +Areas have no end date — sorted by responsibility, not deadline. + +## Areas +- // no document, yet + +## Related +- [Projects Hub](project/index){"type":"related","link-info":"Links to the sibling Projects hub"} +- [Resources Hub](resource/index){"type":"related","link-info":"Links to the sibling Resources hub"} diff --git a/03_resources/.gitkeep b/03_resources/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/03_resources/index.md b/03_resources/index.md new file mode 100644 index 0000000..16d234e --- /dev/null +++ b/03_resources/index.md @@ -0,0 +1,22 @@ +--- +id: resource/index +title: Resources Hub +type: hub +status: active +tags: [resource, hub] +last_reviewed: 2026-09-11 +--- + +# Resources Hub + +Reference material and ingested external resources, one folder each under +`03_resources/`. An ingested resource keeps its original bytes in +`03_resources//raw/` (excluded from the graph) next to the notes derived +from it. + +## Resources +- // no document, yet + +## Related +- [Projects Hub](project/index){"type":"related","link-info":"Links to the sibling Projects hub"} +- [Areas Hub](area/index){"type":"related","link-info":"Links to the sibling Areas hub"} diff --git a/04_archives/README.md b/04_archives/README.md new file mode 100644 index 0000000..0b9ebf1 --- /dev/null +++ b/04_archives/README.md @@ -0,0 +1,20 @@ +--- +id: archive/index +title: Archives +type: hub +status: active +tags: [archive, para] +last_reviewed: 2026-09-11 +--- + +# Archives + +Inactive items moved out of `01_projects/`, `02_areas/`, or `03_resources/` when +they stop being live. Keep notes here for reference and history; do not delete. + +Move a note here by relocating its file and setting `status: deprecated` (or +leaving it as-is for a completed project). Fix any links that break, then run +`make graph`. + +## Archived +- // no document, yet diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..266a557 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1 @@ +always read AGENT_QUERY_GUIDE.md diff --git a/AGENT_QUERY_GUIDE.md b/AGENT_QUERY_GUIDE.md new file mode 100644 index 0000000..9f54756 --- /dev/null +++ b/AGENT_QUERY_GUIDE.md @@ -0,0 +1,108 @@ +--- +id: meta/agent-query-guide +title: Agent Query Guide +type: meta +status: active +tags: [agents, jq, graph] +last_reviewed: 2026-06-16 +--- + +# Agent Query Guide + +`index/graph.json` is for navigation. You can query tags or types there. Use it! + +Schema: + +- Top-level object: + - `version` (number): schema version + - `generated_at` (string): ISO-8601 UTC timestamp + - `nodes` (array): note metadata records + - `edges` (array): directed wiki-link records +- Node object (`nodes[]`): + - `id` (string): stable note identifier (usually from frontmatter) + - `title` (string): note title + - `path` (string): vault-relative markdown path + - `repo` (string): repo scope (`tenant-controller`, `controller-util`, `global`, ...) + - `type` (string): note classification (`guideline`, `policy`, `project`, etc.) + - `status` (string): lifecycle state (`active`, `draft`, `deprecated`) + - `tags` (string array): topic tags + - `last_reviewed` (string): `YYYY-MM-DD` + - `outbound_links` (string array): linked note IDs from this note + - `inbound_count` (number): number of edges targeting this note +- Edge object (`edges[]`): + - `from` (string): source note ID + - `to` (string): target note ID + - `kind` (string): link *mechanism* — normally `markdown_link` (legacy + `wiki_link` is rejected by `--validate` and should not appear in a valid graph) + - `type` (string): semantic *relationship* (see vocabulary below); default `related` + - `anchor_text` (string): display / alias text used in the link + - `annotation` (string): free-text "what the source needs from the destination" + (from the link's `link-info`, falling back to a trailing `— gloss`); may be empty + - `weight` (number): link weight, currently `1.0` + +## Link syntax + +Links are standard **markdown links** whose target is the stable note `id`, with an +optional JSON brace suffix carrying the relationship and annotation: + +```md +[display text](resource/old-deadlines){"type":"supersedes","link-info":"the deadline table this replaces"} +``` + +- A bare `[text](id)` with no brace is a `related` edge. +- `type` must be one of the controlled vocabulary; `generate-graph.py --validate` + fails on any other value: + `related, supersedes, superseded-by, depends-on, part-of, parent, derived-from, + see-also, source`. +- Legacy `[[wiki links]]` are **no longer accepted**: `generate-graph.py --validate` + fails on any that remain. Locate and convert them with + `python3 scripts/list-wiki-links.py`. + +Examples with `jq`: + +```bash +# list repos +jq -r '.nodes[].repo' index/graph.json | sort -u + +# list top linked notes +jq -r '.nodes | sort_by(-(.inbound_count // 0))[] | "\(.inbound_count // 0)\t\(.id)\t\(.path)"' index/graph.json + +# notes in one repo +jq -r '.nodes[] | select(.repo=="tenant-controller") | "\(.id)\t\(.path)"' index/graph.json + +# all projects (PARA category 1) +jq -r '.nodes[] | select(.type=="project") | "\(.id)\t\(.path)"' index/graph.json + +# outgoing links from one note +NOTE_ID="repo/controller-util/info" +jq -r --arg id "$NOTE_ID" '.nodes[] | select(.id==$id) | .outbound_links[]?' index/graph.json + +# incoming links to one note +NOTE_ID="repo/controller-util/info" +jq -r --arg id "$NOTE_ID" '.edges[] | select(.to==$id) | .from' index/graph.json | sort -u + +# outbound edges of a note WITH relationship type + annotation +jq -r --arg id "$NOTE_ID" '.edges[] | select(.from==$id) | "\(.type)\t\(.to)\t\(.annotation)"' index/graph.json + +# all edges of a given relationship type (e.g. what supersedes what) +jq -r '.edges[] | select(.type=="supersedes") | "\(.from) -> \(.to)"' index/graph.json + +# what a note supersedes +jq -r --arg id "$NOTE_ID" '.edges[] | select(.from==$id and .type=="supersedes") | .to' index/graph.json +``` + +For `jg`, use the same filters if your local `jg` is jq-compatible. + +Validation: + +```bash +# validate: non-zero exit on unresolved links, edge types outside the +# vocabulary, OR any remaining legacy [[wiki links]] +python3 scripts/generate-graph.py --validate + +# regenerate graph and validate +python3 scripts/generate-graph.py && python3 scripts/generate-graph.py --validate + +# list any remaining legacy [[wiki links]] (rejected by --validate) to convert +python3 scripts/list-wiki-links.py +``` diff --git a/CLAUDE.md b/CLAUDE.md new file mode 120000 index 0000000..47dc3e3 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1 @@ +AGENTS.md \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..1ddbba5 --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +PYTHON ?= python3 +CLAUDE_HOME ?= $(HOME)/.claude +MCP_NAME ?= knowledge-database + +.PHONY: test generate-graph validate-graph graph install-hooks install-global-claude install-mcp-claude + +test: + $(PYTHON) -m unittest discover -s tests + +generate-graph: + $(PYTHON) scripts/generate-graph.py + +validate-graph: + $(PYTHON) scripts/generate-graph.py --validate + +graph: generate-graph validate-graph + +install-hooks: + chmod +x scripts/pre-commit + ln -sf ../../scripts/pre-commit .git/hooks/pre-commit + +# Link this vault's skills and agents into the global Claude context +# (~/.claude/skills, ~/.claude/agents) so they are invocable from any repo. +install-global-claude: + mkdir -p "$(CLAUDE_HOME)/skills" "$(CLAUDE_HOME)/agents" + @for d in skills/*/; do \ + name=$$(basename "$$d"); \ + [ -f "$$d/SKILL.md" ] || continue; \ + ln -sfn "$(CURDIR)/$$d" "$(CLAUDE_HOME)/skills/$$name"; \ + echo "skill -> $(CLAUDE_HOME)/skills/$$name"; \ + done + @for f in agents/*.md; do \ + name=$$(basename "$$f"); \ + [ "$$name" = "README.md" ] && continue; \ + ln -sfn "$(CURDIR)/$$f" "$(CLAUDE_HOME)/agents/$$name"; \ + echo "agent -> $(CLAUDE_HOME)/agents/$$name"; \ + done + +# Register scripts/mcp-server.py as an MCP server in Claude Code (user scope, +# so it is available from any repo). Re-run to update; removes stale entry first. +install-mcp-claude: + -claude mcp remove --scope user "$(MCP_NAME)" 2>/dev/null + claude mcp add --scope user "$(MCP_NAME)" -- $(PYTHON) "$(CURDIR)/scripts/mcp-server.py" + @echo "mcp -> $(MCP_NAME) ($(CURDIR)/scripts/mcp-server.py)" diff --git a/README.md b/README.md new file mode 100644 index 0000000..0dfe31c --- /dev/null +++ b/README.md @@ -0,0 +1,63 @@ +--- +id: knowledge-base/home +title: Knowledge Base +type: hub +status: active +tags: [knowledge-base, agents, docs, para] +last_reviewed: 2026-09-11 +--- + +# Knowledge Base + +A Markdown knowledge graph for engineering agents, organized with the +[PARA method](https://fortelabs.com/blog/para/) — notes live by **actionability**, +not by topic. + +This repository is the empty **framework**: tooling, agents, skills and the PARA +skeleton, with no notes in it. Clone it, run the bootstrap below, and start +writing notes. + +## Layout + +``` +01_projects/ active efforts with a goal and an end (features in flight) +02_areas/ ongoing responsibilities & standards (policies, value streams, decisions) +03_resources/ reference material (repo info, ingested externals) +04_archives/ inactive items moved out of the three live categories +index/ generated graph (index/graph.json) +scripts/ graph generator, link inventory, MCP server, git hook +agents/ subagent definitions, symlinked into .claude/agents/ +skills/ skill definitions, symlinked into .claude/skills/ +tests/ unit tests for the tooling +``` + +The PARA buckets start empty — add notes per the layout above, each in its own +folder with an `index.md`. + +## Bootstrap + +```sh +make install-hooks # pre-commit hook: regenerate + validate the graph +make install-global-claude # link skills & agents into ~/.claude so any repo can use them +make install-mcp-claude # register scripts/mcp-server.py as the knowledge-database MCP server +make graph # generate index/graph.json and validate it +make test # run the tooling tests +``` + +## Generation +- Run `make graph` +- Output is written to `index/graph.json` + +## Rules +- Markdown is the source of truth +- `index/graph.json` is derived and generated +- Keep notes linked using typed markdown links: `[text](note-id){"type":"…"}` (legacy `[[wiki links]]` are rejected by `make validate-graph`) +- Organize by PARA actionability; move notes between buckets as that changes +- Fix broken links — find them with `make graph` or `make validate-graph` + +## Hubs +- [Projects Hub](project/index){"type":"related","link-info":"Lists the active projects"} +- [Areas Hub](area/index){"type":"related","link-info":"Lists the ongoing areas of responsibility"} +- [Resources Hub](resource/index){"type":"related","link-info":"Lists the cataloged reference notes"} +- [Archives](archive/index){"type":"related","link-info":"Lists items moved out of the live buckets"} +- [Agent Query Guide](meta/agent-query-guide){"type":"see-also","link-info":"Graph schema and jq queries for agents"} diff --git a/agents/README.md b/agents/README.md new file mode 100644 index 0000000..c7fe872 --- /dev/null +++ b/agents/README.md @@ -0,0 +1,23 @@ +# Agents + +Subagent definitions for this vault. Mirrors the `skills/` convention: each agent +lives here and is wired into `.claude/agents/` as a relative symlink +(`.claude/agents/.md -> ../../agents/.md`). + +- `the-knowing-one.md` — read-only information gatherer. Send it a topic or task; + it sweeps the whole vault (notes, policies, ADRs, repo info, areas, projects, + the graph index) and follows links — internal typed markdown links and external + URLs — to return a cited, synthesized briefing. +- `the-scribe.md` — MCP-only resource ingestor. Wraps the + `knowledge-base-ingest-resource` skill with no shell: it captures the original + bytes (`capture_raw`, local file or download, binary included), writes notes, + refreshes the graph, commits the named paths, and pushes entirely through the + `knowledge-database` MCP server. + +When creating a new agent or renaming a file, adjust the symlinks in +`./.claude/agents/`. To use an agent from another agent's home dir, link it like +the skills, e.g.: + +```sh +ln -s "$PWD/agents/the-knowing-one.md" ~/.claude/agents/the-knowing-one.md +``` diff --git a/agents/the-knowing-one.md b/agents/the-knowing-one.md new file mode 100644 index 0000000..811180a --- /dev/null +++ b/agents/the-knowing-one.md @@ -0,0 +1,135 @@ +--- +name: the-knowing-one +description: Information gatherer for the knowledge-base vault (personal docs-as-code knowledge database, PARA-organized). Send it a topic or task; it sweeps the whole vault via the knowledge-database MCP server — notes, policies, ADRs, repo info, areas, projects, the graph index — AND follows links (internal typed markdown links and external URLs) to pull in related context. Returns a cited, synthesized briefing. Read-only. Use when you need to "know everything the vault has" about something before acting. +tools: WebFetch, WebSearch, ToolSearch, mcp__knowledge-database__* +mcpServers: + - knowledge-database +--- + +# The Knowing One + +You are **The Knowing One** — sent to gather everything the +knowledge-base vault knows about a given topic or task, and to +chase the links outward to related context. You report; you do not change. +**Read-only: never edit, create, or commit notes.** + +**The knowledge-database MCP server is your ONLY way into the vault.** Do not +read files from disk, run shell commands, run `jq`/`grep`, or clone the repo — +you have no tools to do so. Every vault fact comes from an MCP tool call. The +only non-MCP tools you may use are `WebFetch`/`WebSearch`, and ONLY to follow +external URLs that notes cite — never to reach the vault itself. + +The vault is a personal docs-as-code knowledge database, +organized with the PARA method. Markdown notes are the source of +truth; a generated graph index navigates them. You access both exclusively +through MCP. + +## MCP tools + +- `graph_summary` — overview of the graph: counts, types, tags, central nodes. +- `graph_search_nodes` — find nodes by tag / type / repo / id / text. +- `graph_neighbors` — outbound + inbound links for a node id, each with its edge + `type` + `annotation` (traverse the graph). +- `read_note` — the actual Markdown of a note; the ANSWER lives here, not in the index. +- `refresh_graph` — regenerate the graph index if it looks stale (read-only; changes no Markdown). + +## Mandate + +Given a **topic or task**, return a complete, cited briefing: + +1. **Orient** — what the vault holds and where the topic likely lives (`graph_summary`). +2. **Find** — every relevant note, via `graph_search_nodes` (+ text-search fallback). +3. **Read** — the actual Markdown via `read_note`; the answer is there, not in the index. +4. **Follow links** — traverse internal links (via `graph_neighbors`) AND external URLs. +5. **Synthesize** — one briefing, every claim cited to a note path/id. + +## Vault layout (PARA) + +``` +01_projects/ active efforts with a goal and an end +02_areas/ ongoing standards: policies, value-stream areas, decisions (ADRs) +03_resources/meta/ taxonomy + note templates +03_resources/repos// per-repo info.md (+ decisions/) +03_resources/repos/repo-landscape.md index of documented repos +04_archives/ inactive items +README.md vault hub +``` + +Every note carries taxonomy frontmatter: `id`, `title`, `type` +(`hub|project|guideline|policy|feature|roadmap|decision|note|meta`), `status` +(`active|draft|deprecated`), `tags`, `last_reviewed`, optional `repo`. Notes link +with typed markdown links — `[text](note-id){"type":"…"}` — resolved by `id`. +Legacy `[[wiki links]]` are rejected by the graph validator and must not be written. + +## 1. Orient + +Start broad with `graph_summary` to see node counts, types, tags, and the +most-central notes. Then `read_note` the entry points: + +- vault hub (`README.md`) — entry points. +- `03_resources/repos/repo-landscape.md` — index of documented repos, grouped. +- `02_areas/ways-of-working.md` — global engineering policy. +- `03_resources/meta/taxonomy.md` — frontmatter fields and PARA buckets. + +For a repo topic, `read_note` on `03_resources/repos//info.md`. If +`graph_search_nodes` returns no node for it, say it is not yet documented. + +## 2. Find — query the graph + +Use `graph_search_nodes` to locate notes: + +- by **repo** — every note tied to a repo (e.g. `tenant-controller`). +- by **type** — `policy`, `decision`, `feature`, `guideline`, `project`, ... +- by **tag** — e.g. `kcp`. +- by **text** — when tag/type/id unknown, full-text search the notes. + +Use `graph_summary` to spot central (most-linked) notes worth reading early. + +If the graph looks stale or a node/path seems missing, call `refresh_graph` +(read-only) and re-query. + +## 3. Read the source notes + +The graph gives ids + paths; the ANSWER is in the Markdown. Call `read_note` on +every matched node and read it. + +## 4. Follow the links — internal AND external + +Links are the intended navigation. Traverse, do not guess. + +- **Internal links** (typed markdown `[text](id){"type":"…"}`) — call + `graph_neighbors` on each core node to get outbound + inbound links, then + `read_note` the related policies, decisions, repo info. Each neighbor carries an + edge `type` (relationship: supersedes, depends-on, part-of, …) and an + `annotation` (what to expect in that note) — use them to prioritize which hops + are worth following. Walk both directions: who this note links to, and who links + to it. Go 1–2 hops out from the core matches; stop when notes stop being + relevant. Avoid cycles — track visited ids. +- **External URLs** — when a note cites an external source (repo URL, docs, spec, + ticket), fetch it with `WebFetch` to pull relevant facts. Use `WebSearch` only + to locate an official source a note refers to but does not link. Keep external + reads scoped to the topic. + +## 5. Report — synthesized + cited + +- Answer the topic/task directly from what the notes say. +- Cite every claim by vault-relative `path` (e.g. + `03_resources/repos/controller-util/info.md`) and/or `id`. +- Separate vault facts from external-source facts (give the URL) and from your + own inference. +- Flag `status: draft`/`deprecated` notes. If the vault does not cover part of + the topic, say so plainly — do not invent. +- List the link trail you followed (internal hops with their edge `type`, plus + external URLs) so the requester can verify and go deeper. + +Your final message IS the briefing returned to the caller — make it +self-contained: findings, citations, gaps, link trail. + +## Gaps — suggest a collector + +When the vault does not cover part of the topic, or a note cites an external +source worth capturing, you MAY ask the user to start **The Collector** (the +ingestion agent) to pull that resource into the vault. **This requires explicit +user permission** — you never start a collector yourself, and you never ingest +or write anything (you are read-only). Surface the gap, name the resource +(file path / URL / video) that would fill it, and let the user decide. diff --git a/agents/the-scribe.md b/agents/the-scribe.md new file mode 100644 index 0000000..bb0f18e --- /dev/null +++ b/agents/the-scribe.md @@ -0,0 +1,112 @@ +--- +name: the-scribe +description: MCP-only resource ingestor for the knowledge-base vault (PARA-organized). Hand it a URL, a local file path, or a PDF and it runs the `knowledge-base-ingest-resource` pipeline entirely through the `knowledge-database` MCP server — captures the original bytes (text or binary) into `raw/`, writes the resource note and one note per topic, wires typed markdown links, regenerates + validates the graph, then commits the named paths and pushes. Has no shell. Use when the user says "ingest", "add this resource", or "save this link/paper" and wants the result written and published. +tools: Read, WebFetch, Skill, mcp__knowledge-database__* +mcpServers: + - knowledge-database +--- + +# The Scribe + +You are **The Scribe** — you ingest an external resource into the knowledge-base +vault and publish the result. One resource per invocation. You write what the +source says; you never invent content. + +The resource comes from the calling context — a URL, a pasted link, or a +description. If none is given, ask for one. + +**The `knowledge-database` MCP server is your ONLY way to change the vault.** You +have no Bash, no Write/Edit, no Grep/Glob. Every write, graph run, commit, and +push is an MCP tool call. `WebFetch` fetches the external resource itself, never +the vault. `Read` is allowed for exactly one job: opening a **binary** file you +already captured into `raw/` (a PDF, an image) at the absolute path +`capture_raw` returns — for vault Markdown, use `read_note`. + +## MCP tools + +Read: +- `graph_summary` — counts, types, tags, most-linked notes. +- `graph_search_nodes` — find notes by tag / type / repo / id / free text. +- `graph_neighbors` — inbound + outbound links of a node, with edge `type` and `annotation`. +- `read_note` — the Markdown of a note (by `id` or vault-relative `path`). On a + binary file it returns `binary: true` plus a hint with the absolute path + instead of garbled text — open that path with `Read`. + +Write: +- `write_note(path, content, overwrite?)` — create a **text** note; parent dirs + are made for you. Existing files need `overwrite=true`. +- `capture_raw(path, source_path | url, overwrite?, max_bytes?)` — copy a local + file or download an http(s) URL into the vault **byte-for-byte**. This is how + binaries (PDF, image, archive) land in `raw/`. Returns `absolute_path`, + `bytes`, `sha256`. Default ceiling 100 MiB. +- `edit_note(path, old_string, new_string, replace_all?)` — exact-string edit, + for backlinks into notes that already exist. + +Publish: +- `refresh_graph(validate=true)` — regenerate and validate `index/graph.json`. +- `git_commit(message, paths[])` — stage the named paths and commit. +- `git_push(remote?, branch?)` — push to `origin` / current branch. + +## Mandate + +Invoke the skill first, then follow it step by step: + +``` +Skill(knowledge-base-ingest-resource) +``` + +The skill is the source of truth for the procedure. Translate each of its shell +steps into the MCP equivalent: + +| Skill step | Your call | +|-------------------------------------|--------------------------------------------------| +| check slug not taken / find matches | `graph_search_nodes`, `read_note` | +| `printf … > raw/source.url` | `write_note("03_resources//raw/source.url", …)` | +| save fetched page to `raw/page.md` | `WebFetch` → `write_note(".../raw/page.md", …)` | +| `cp "" raw/` | `capture_raw(path=".../raw/", source_path="")` | +| `curl -fsSL "" -o raw/` | `capture_raw(path=".../raw/", url="")` | +| read the captured binary | `Read()` | +| write the topic + index notes | `write_note` | +| backlink an existing note | `edit_note` | +| `make graph` | `refresh_graph(validate=true)` | +| `git add ` (one by one) | `git_commit(message, paths=[…])` | + +## Operating rules + +- **Preserve the source, bytes first.** Capture into `03_resources//raw/` + before writing any note. Pick by resource type: + - **Local file (any type)** — `capture_raw(source_path=…)`, keeping the original + filename. + - **PDF or other binary at a URL** — `capture_raw(url=…)`, then `Read` the + returned `absolute_path` to get at the content. + - **Webpage** — `write_note(".../raw/source.url", "\n")` plus the `WebFetch` + body into `raw/page.md`. + Never fabricate a page or transcript — if the fetch or capture fails, stop and + report what is missing. Verify the capture landed: the result carries `bytes` + and `sha256`. +- **YouTube is still out of reach.** Transcripts need `yt-dlp` or + `youtube-transcript-api`, which are shell tools you do not have. Capture the + URL into `raw/source.url`, then hand the video to **the-collector** rather than + inventing transcript text. +- **Link, don't duplicate.** Search the graph before writing a topic note. If the + vault already covers the topic, link it instead of creating a second note. +- **Typed markdown links only.** `[title](note-id){"type":"…","link-info":"…"}`. + `type` is from the controlled vocabulary — `related, supersedes, + superseded-by, depends-on, part-of, parent, derived-from, see-also, source`. + Legacy `[[wiki links]]` fail validation. +- **Never write the graph.** `index/graph.json` is derived; `write_note` refuses + it. Use `refresh_graph`. Fix every unresolved link until validation is clean — + the pre-commit hook rejects a stale or invalid graph. +- **Name every path you commit.** `git_commit` takes an explicit list; wildcard + staging is rejected by the server. Include the resource folder, every + backlinked note, and `index/graph.json`. +- **Push straight to `main`.** This vault takes ingests as direct commits; no + branch, no PR. Commit with Conventional Commits (`feat: ingest `), + then `git_push`. If the caller says "don't push", stop after the commit. + +## Report + +Return a self-contained summary: resource id/path and type, what landed in +`raw/` (with size + sha256 for captured binaries), the topic notes created, which topics linked to existing notes vs. were +captured as external URLs vs. left as candidates, every path committed, the +graph-validation result, and the commit + push outcome. diff --git a/index/.gitkeep b/index/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/index/graph.json b/index/graph.json new file mode 100644 index 0000000..b670de3 --- /dev/null +++ b/index/graph.json @@ -0,0 +1,347 @@ +{ + "version": 1, + "generated_at": "2026-09-11T07:34:35.807809+00:00", + "nodes": [ + { + "id": ".claude/agents/the-knowing-one", + "title": "The Knowing One", + "path": ".claude/agents/the-knowing-one.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": ".claude/agents/the-scribe", + "title": "The Scribe", + "path": ".claude/agents/the-scribe.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "AGENTS", + "title": "Agents", + "path": "AGENTS.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "CLAUDE", + "title": "Claude", + "path": "CLAUDE.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "GLOBAL_AGENTS", + "title": "GLOBAL_AGENTS.md", + "path": "GLOBAL_AGENTS.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "agents/README", + "title": "Agents", + "path": "agents/README.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "agents/the-knowing-one", + "title": "The Knowing One", + "path": "agents/the-knowing-one.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "agents/the-scribe", + "title": "The Scribe", + "path": "agents/the-scribe.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "archive/index", + "title": "Archives", + "path": "04_archives/README.md", + "repo": "global", + "type": "hub", + "status": "active", + "tags": [ + "archive", + "para" + ], + "last_reviewed": "2026-09-11", + "outbound_links": [], + "inbound_count": 1 + }, + { + "id": "area/index", + "title": "Areas Hub", + "path": "02_areas/index.md", + "repo": "global", + "type": "hub", + "status": "active", + "tags": [ + "area", + "hub" + ], + "last_reviewed": "2026-09-11", + "outbound_links": [ + "project/index", + "resource/index" + ], + "inbound_count": 3 + }, + { + "id": "knowledge-base/home", + "title": "Knowledge Base", + "path": "README.md", + "repo": "global", + "type": "hub", + "status": "active", + "tags": [ + "knowledge-base", + "agents", + "docs", + "para" + ], + "last_reviewed": "2026-09-11", + "outbound_links": [ + "archive/index", + "area/index", + "meta/agent-query-guide", + "project/index", + "resource/index" + ], + "inbound_count": 0 + }, + { + "id": "meta/agent-query-guide", + "title": "Agent Query Guide", + "path": "AGENT_QUERY_GUIDE.md", + "repo": "global", + "type": "meta", + "status": "active", + "tags": [ + "agents", + "jq", + "graph" + ], + "last_reviewed": "2026-06-16", + "outbound_links": [], + "inbound_count": 1 + }, + { + "id": "project/index", + "title": "Projects Hub", + "path": "01_projects/index.md", + "repo": "global", + "type": "hub", + "status": "active", + "tags": [ + "project", + "hub" + ], + "last_reviewed": "2026-09-11", + "outbound_links": [ + "area/index", + "resource/index" + ], + "inbound_count": 3 + }, + { + "id": "resource/index", + "title": "Resources Hub", + "path": "03_resources/index.md", + "repo": "global", + "type": "hub", + "status": "active", + "tags": [ + "resource", + "hub" + ], + "last_reviewed": "2026-09-11", + "outbound_links": [ + "area/index", + "project/index" + ], + "inbound_count": 3 + }, + { + "id": "skills/README", + "title": "Readme", + "path": "skills/README.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "skills/knowledge-base-ingest-resource/SKILL", + "title": "Ingest a Resource", + "path": "skills/knowledge-base-ingest-resource/SKILL.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + }, + { + "id": "skills/knowledge-base-read/SKILL", + "title": "Read the Knowledge Base", + "path": "skills/knowledge-base-read/SKILL.md", + "repo": "global", + "type": "note", + "status": "active", + "tags": [], + "last_reviewed": "", + "outbound_links": [], + "inbound_count": 0 + } + ], + "edges": [ + { + "from": "area/index", + "to": "project/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Projects Hub", + "annotation": "Links to the sibling Projects hub", + "weight": 1.0 + }, + { + "from": "area/index", + "to": "resource/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Resources Hub", + "annotation": "Links to the sibling Resources hub", + "weight": 1.0 + }, + { + "from": "knowledge-base/home", + "to": "archive/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Archives", + "annotation": "Lists items moved out of the live buckets", + "weight": 1.0 + }, + { + "from": "knowledge-base/home", + "to": "area/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Areas Hub", + "annotation": "Lists the ongoing areas of responsibility", + "weight": 1.0 + }, + { + "from": "knowledge-base/home", + "to": "meta/agent-query-guide", + "kind": "markdown_link", + "type": "see-also", + "anchor_text": "Agent Query Guide", + "annotation": "Graph schema and jq queries for agents", + "weight": 1.0 + }, + { + "from": "knowledge-base/home", + "to": "project/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Projects Hub", + "annotation": "Lists the active projects", + "weight": 1.0 + }, + { + "from": "knowledge-base/home", + "to": "resource/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Resources Hub", + "annotation": "Lists the cataloged reference notes", + "weight": 1.0 + }, + { + "from": "project/index", + "to": "area/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Areas Hub", + "annotation": "Links to the sibling Areas hub", + "weight": 1.0 + }, + { + "from": "project/index", + "to": "resource/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Resources Hub", + "annotation": "Links to the sibling Resources hub", + "weight": 1.0 + }, + { + "from": "resource/index", + "to": "area/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Areas Hub", + "annotation": "Links to the sibling Areas hub", + "weight": 1.0 + }, + { + "from": "resource/index", + "to": "project/index", + "kind": "markdown_link", + "type": "related", + "anchor_text": "Projects Hub", + "annotation": "Links to the sibling Projects hub", + "weight": 1.0 + } + ] +} diff --git a/scripts/generate-graph.py b/scripts/generate-graph.py new file mode 100644 index 0000000..1b3e87d --- /dev/null +++ b/scripts/generate-graph.py @@ -0,0 +1,499 @@ +#!/usr/bin/env python3 +import argparse +import json +import re +import sys +from collections import defaultdict +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Dict, List, Tuple + + +# Obsidian-style wiki link: [[target#anchor|alias]] +WIKI_LINK_RE = re.compile(r"\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|([^\]]+))?\]\]") +# Standard markdown link: [display](target); (? edge anchor_text + kind: str # "wiki_link" | "markdown_link" + rel_type: str # semantic relationship type (default "related") + annotation: str # per-edge "what to expect there" note + lineno: int # 1-based line number in the note body + raw: str # the raw link text as written (for the inventory script) + is_external: bool # markdown link whose target is a URL/anchor -> no edge + + +@dataclass +class Note: + path: Path + rel_path: str + stem: str + meta: Dict[str, object] + title: str + note_id: str + links: List[LinkRef] + + +@dataclass(frozen=True) +class UnresolvedLink: + from_id: str + from_path: str + target: str + + +@dataclass(frozen=True) +class WikiLinkUse: + from_id: str + from_path: str + target: str + lineno: int + raw: str + + +def parse_frontmatter(text: str) -> Tuple[Dict[str, object], str]: + if not text.startswith("---\n"): + return {}, text + + end = text.find("\n---\n", 4) + if end == -1: + return {}, text + + fm_block = text[4:end] + body = text[end + 5 :] + meta: Dict[str, object] = {} + + for raw in fm_block.splitlines(): + line = raw.strip() + if not line or line.startswith("#") or ":" not in line: + continue + key, value = line.split(":", 1) + key = key.strip() + value = value.strip() + if value.startswith("[") and value.endswith("]"): + inner = value[1:-1].strip() + if inner: + meta[key] = [v.strip().strip('"').strip("'") for v in inner.split(",")] + else: + meta[key] = [] + else: + meta[key] = value.strip('"').strip("'") + + return meta, body + + +def derive_title(meta: Dict[str, object], body: str, stem: str) -> str: + if isinstance(meta.get("title"), str) and meta["title"].strip(): + return str(meta["title"]).strip() + for line in body.splitlines(): + if line.startswith("# "): + return line[2:].strip() + return stem.replace("-", " ").strip().title() + + +def mask_code(body: str) -> str: + """Blank out code so links inside code samples are ignored, preserving line + boundaries so line-based parsing (gloss fallback, line numbers) stays aligned.""" + body = FENCED_CODE_RE.sub(lambda m: "\n" * m.group(0).count("\n"), body) + body = INLINE_CODE_RE.sub(lambda m: " " * len(m.group(0)), body) + return body + + +def scan_brace(text: str, start: int) -> Tuple[object, int]: + """Given text[start] == '{', scan to the matching '}' respecting JSON string + quoting, and json.loads the block. Returns (obj_or_None, end_index_after_brace). + On an unterminated or malformed brace, returns (None, start) so the caller + treats it as no brace.""" + depth = 0 + in_str = False + esc = False + i = start + while i < len(text): + c = text[i] + if in_str: + if esc: + esc = False + elif c == "\\": + esc = True + elif c == '"': + in_str = False + elif c == '"': + in_str = True + elif c == "{": + depth += 1 + elif c == "}": + depth -= 1 + if depth == 0: + raw = text[start : i + 1] + try: + return json.loads(raw), i + 1 + except (json.JSONDecodeError, ValueError): + return None, start + i += 1 + return None, start + + +def clean_md_target(raw: str) -> str: + """Normalize a markdown link target: drop a trailing ``"title"`` and <> wrappers.""" + raw = raw.strip() + if raw.startswith("<") and raw.endswith(">"): + raw = raw[1:-1].strip() + # `[text](target "title")` -> target + for sep in (' "', " '"): + if sep in raw: + raw = raw.split(sep, 1)[0].strip() + return raw + + +def parse_links(body: str) -> List[LinkRef]: + masked = mask_code(body) + refs: List[LinkRef] = [] + + for lineno, line in enumerate(masked.splitlines(), start=1): + # Collect wiki + markdown link matches on this line, in order. + raw_matches: List[Tuple[int, int, str, str, str, str]] = [] + for m in WIKI_LINK_RE.finditer(line): + target = m.group(1).strip() + alias = (m.group(2) or target).strip() + if target: + raw_matches.append( + (m.start(), m.end(), "wiki_link", target, alias, m.group(0)) + ) + for m in MD_LINK_RE.finditer(line): + target = clean_md_target(m.group(2)) + display = m.group(1).strip() + if target: + raw_matches.append( + ( + m.start(), + m.end(), + "markdown_link", + target, + display or target, + m.group(0), + ) + ) + raw_matches.sort(key=lambda x: x[0]) + + line_refs: List[Tuple[LinkRef, int]] = [] # (ref, end-index-after-brace) + for start, end, kind, target, anchor, raw in raw_matches: + rel_type, annotation = "related", "" + brace_end = end + if end < len(line) and line[end] == "{": + obj, be = scan_brace(line, end) + if isinstance(obj, dict): + brace_end = be + t = obj.get("type") + if isinstance(t, str) and t.strip(): + rel_type = t.strip() + info = obj.get("link-info") + if isinstance(info, str) and info.strip(): + annotation = info.strip() + is_external = kind == "markdown_link" and bool( + EXTERNAL_TARGET_RE.match(target) + ) + ref = LinkRef( + target=target, + anchor=anchor, + kind=kind, + rel_type=rel_type, + annotation=annotation, + lineno=lineno, + raw=raw, + is_external=is_external, + ) + line_refs.append((ref, brace_end)) + + # Annotation fallback: a trailing "— gloss" applies only when the link is + # the sole link on its line (so we don't mis-assign a shared gloss). + if len(line_refs) == 1: + ref, brace_end = line_refs[0] + if not ref.annotation: + gloss = GLOSS_RE.match(line[brace_end:]) + if gloss: + ref.annotation = gloss.group(1).strip() + + refs.extend(ref for ref, _ in line_refs) + + return refs + + +def norm(s: str) -> str: + return s.strip().lower() + + +def build_notes(root: Path) -> List[Note]: + notes: List[Note] = [] + for p in sorted(root.rglob("*.md")): + rel = p.relative_to(root) + if rel.parts and rel.parts[0] in {"index", "scripts", ".obsidian"}: + continue + # `raw/` holds untouched ingested originals (external READMEs, specs, …), + # not vault notes — skip them so their file-relative markdown links are not + # treated as (broken) graph edges. + if "raw" in rel.parts: + continue + + text = p.read_text(encoding="utf-8") + meta, body = parse_frontmatter(text) + title = derive_title(meta, body, p.stem) + note_id = str(meta.get("id") or rel.with_suffix("")).strip() + links = parse_links(body) + # parse_links numbers lines within `body`; shift to absolute file line + # numbers by the number of lines the frontmatter block occupied. + line_offset = text.count("\n", 0, len(text) - len(body)) + if line_offset: + for ref in links: + ref.lineno += line_offset + + notes.append( + Note( + path=p, + rel_path=rel.as_posix(), + stem=p.stem, + meta=meta, + title=title, + note_id=note_id, + links=links, + ) + ) + return notes + + +def build_alias_map(notes: List[Note]) -> Dict[str, str]: + alias_map: Dict[str, str] = {} + stem_buckets: Dict[str, List[str]] = defaultdict(list) + + for n in notes: + stem_buckets[norm(n.stem)].append(n.note_id) + + for n in notes: + alias_map[norm(n.note_id)] = n.note_id + alias_map[norm(n.rel_path)] = n.note_id + alias_map[norm(Path(n.rel_path).with_suffix("").as_posix())] = n.note_id + alias_map[norm(n.title)] = n.note_id + + for stem, ids in stem_buckets.items(): + if len(ids) == 1: + alias_map[stem] = ids[0] + + return alias_map + + +def generate_graph( + root: Path, existing_generated_at: str = "" +) -> Tuple[Dict[str, object], List[UnresolvedLink], List[WikiLinkUse]]: + notes = build_notes(root) + alias_map = build_alias_map(notes) + + edges_set = set() + outbound: Dict[str, List[str]] = defaultdict(list) + inbound_count: Dict[str, int] = defaultdict(int) + unresolved_set = set() + wiki_uses_set = set() + + for n in notes: + for ref in n.links: + # Legacy [[wiki links]] are still parsed (so we can flag them) but are + # no longer an accepted link form — --validate rejects any that remain. + if ref.kind == "wiki_link": + wiki_uses_set.add( + WikiLinkUse( + from_id=n.note_id, + from_path=n.rel_path, + target=ref.target, + lineno=ref.lineno, + raw=ref.raw, + ) + ) + if ref.is_external: + continue + target = alias_map.get(norm(ref.target)) + if not target: + unresolved_set.add( + UnresolvedLink( + from_id=n.note_id, + from_path=n.rel_path, + target=ref.target, + ) + ) + continue + edge = (n.note_id, target, ref.kind, ref.rel_type, ref.annotation, ref.anchor) + if edge in edges_set: + continue + edges_set.add(edge) + outbound[n.note_id].append(target) + inbound_count[target] += 1 + + nodes = [] + for n in sorted(notes, key=lambda x: x.note_id): + tags = n.meta.get("tags", []) + if not isinstance(tags, list): + tags = [] + nodes.append( + { + "id": n.note_id, + "title": n.title, + "path": n.rel_path, + "repo": n.meta.get("repo", "global"), + "type": n.meta.get("type", "note"), + "status": n.meta.get("status", "active"), + "tags": tags, + "last_reviewed": n.meta.get("last_reviewed", ""), + "outbound_links": sorted(set(outbound.get(n.note_id, []))), + "inbound_count": inbound_count.get(n.note_id, 0), + } + ) + + edges = [ + { + "from": src, + "to": dst, + "kind": kind, + "type": rel_type, + "anchor_text": anchor, + "annotation": annotation, + "weight": 1.0, + } + for src, dst, kind, rel_type, annotation, anchor in sorted(edges_set) + ] + + graph = { + "version": 1, + "generated_at": existing_generated_at + if existing_generated_at + else datetime.now(timezone.utc).isoformat(), + "nodes": nodes, + "edges": edges, + } + unresolved = sorted(unresolved_set, key=lambda x: (x.from_path, x.target)) + wiki_uses = sorted(wiki_uses_set, key=lambda x: (x.from_path, x.lineno, x.raw)) + return graph, unresolved, wiki_uses + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description="Generate and validate Obsidian graph JSON" + ) + parser.add_argument( + "--validate", + action="store_true", + help="validate links: unresolved targets, edge relationship types, " + "and legacy [[wiki links]] (all rejected)", + ) + parser.add_argument( + "--write", + action="store_true", + help="write index/graph.json (default unless --validate is used alone)", + ) + return parser.parse_args() + + +def print_unresolved(unresolved: List[UnresolvedLink]) -> None: + if not unresolved: + print("Validation passed: no unresolved links") + return + + print(f"Validation failed: {len(unresolved)} unresolved link(s)") + for link in unresolved: + print(f"- {link.from_path}: [[{link.target}]]") + + +def print_wiki_links(wiki_uses: List[WikiLinkUse]) -> None: + if not wiki_uses: + print("Validation passed: no legacy [[wiki links]]") + return + + print(f"Validation failed: {len(wiki_uses)} legacy [[wiki link(s)]]") + print(" use typed markdown links; find/convert them with scripts/list-wiki-links.py") + for w in wiki_uses: + print(f"- {w.from_path}:{w.lineno}: {w.raw}") + + +def find_invalid_types(graph: Dict[str, object]) -> List[Dict[str, object]]: + return [ + e + for e in graph.get("edges", []) # type: ignore[union-attr] + if e.get("type") not in RELATION_TYPES + ] + + +def print_invalid_types(invalid: List[Dict[str, object]]) -> None: + if not invalid: + print("Validation passed: all edge types are in the controlled vocabulary") + return + + print(f"Validation failed: {len(invalid)} edge(s) with unknown relationship type") + print(f" allowed: {', '.join(sorted(RELATION_TYPES))}") + for e in invalid: + print(f"- {e.get('from')} -> {e.get('to')}: type={e.get('type')!r}") + + +def main() -> None: + args = parse_args() + root = Path(__file__).resolve().parents[1] + output = root / "index" / "graph.json" + should_write = args.write or not args.validate + + if should_write: + output.parent.mkdir(parents=True, exist_ok=True) + + existing_generated_at = "" + if output.exists(): + try: + existing = json.loads(output.read_text(encoding="utf-8")) + existing_nodes = existing.get("nodes", []) + existing_edges = existing.get("edges", []) + candidate_graph, unresolved, wiki_uses = generate_graph(root) + if ( + candidate_graph["nodes"] == existing_nodes + and candidate_graph["edges"] == existing_edges + ): + existing_generated_at = str(existing.get("generated_at", "")) + graph, unresolved, wiki_uses = generate_graph( + root, existing_generated_at=existing_generated_at + ) + except (json.JSONDecodeError, OSError, TypeError, ValueError): + graph, unresolved, wiki_uses = generate_graph(root) + else: + graph, unresolved, wiki_uses = generate_graph(root) + + if should_write: + output.write_text(json.dumps(graph, indent=2) + "\n", encoding="utf-8") + print(f"Wrote {output}") + + if args.validate: + print_unresolved(unresolved) + invalid = find_invalid_types(graph) + print_invalid_types(invalid) + print_wiki_links(wiki_uses) + if unresolved or invalid or wiki_uses: + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/list-wiki-links.py b/scripts/list-wiki-links.py new file mode 100755 index 0000000..b3f8066 --- /dev/null +++ b/scripts/list-wiki-links.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +"""List remaining [[wiki links]] so they can be converted to markdown links by hand. + +Legacy `[[wiki links]]` are no longer accepted — `generate-graph.py --validate` +(run by the pre-commit hook) fails on any that remain. This tool locates the +offenders so you can fix them. + +Read-only: this script never edits notes. It reports every `[[wiki link]]` still in +the vault (skipping code samples) with its file, line, resolved target, and the full +line as context, plus a suggested markdown conversion. Convert each link yourself, +adding the right `type` and `link-info`, e.g.: + + [[resource/old-deadlines]] -> [old deadline table](resource/old-deadlines){"type":"supersedes","link-info":"the table this replaces"} + +The relationship `type` must be one of the controlled vocabulary in generate-graph.py +(RELATION_TYPES); the pre-commit hook rejects unknown types. +""" +import argparse +import importlib.util +from collections import defaultdict +from pathlib import Path +from typing import Dict, List + +# Reuse the parser/resolver from generate-graph.py (hyphenated name -> load by path). +_GG_PATH = Path(__file__).resolve().with_name("generate-graph.py") +_spec = importlib.util.spec_from_file_location("generate_graph", _GG_PATH) +gg = importlib.util.module_from_spec(_spec) +assert _spec and _spec.loader +_spec.loader.exec_module(gg) + + +def collect(root: Path): + notes = gg.build_notes(root) + alias_map = gg.build_alias_map(notes) + title_by_id = {n.note_id: n.title for n in notes} + line_cache: Dict[str, List[str]] = {} + + rows = [] + for n in sorted(notes, key=lambda x: x.rel_path): + wiki = [ref for ref in n.links if ref.kind == "wiki_link"] + if not wiki: + continue + lines = line_cache.setdefault( + n.rel_path, n.path.read_text(encoding="utf-8").splitlines() + ) + for ref in wiki: + resolved = alias_map.get(gg.norm(ref.target)) + title = title_by_id.get(resolved, "") if resolved else "" + context = lines[ref.lineno - 1].strip() if ref.lineno - 1 < len(lines) else "" + display = title or (resolved or ref.target) + suggestion = f'[{display}]({resolved or ref.target})' + ( + '{"type":"related","link-info":"…"}' + ) + rows.append( + { + "path": n.rel_path, + "line": ref.lineno, + "raw": ref.raw, + "resolved": resolved or "", + "title": title, + "context": context, + "suggestion": suggestion, + } + ) + return rows + + +def print_grouped(rows) -> None: + by_file: Dict[str, list] = defaultdict(list) + for r in rows: + by_file[r["path"]].append(r) + + for path in sorted(by_file): + items = by_file[path] + print(f"\n{path} ({len(items)})") + for r in items: + print(f" L{r['line']}: {r['raw']} -> {r['resolved']}") + if r["context"]: + print(f" context: {r['context']}") + print(f" suggest: {r['suggestion']}") + + print("\n" + "=" * 60) + print(f"{len(rows)} wiki link(s) remaining across {len(by_file)} file(s)") + if not rows: + print("All links migrated to markdown form. 🎉") + + +def print_tsv(rows) -> None: + print("path\tline\traw\tresolved\ttitle\tcontext") + for r in rows: + ctx = r["context"].replace("\t", " ") + print( + f"{r['path']}\t{r['line']}\t{r['raw']}\t{r['resolved']}\t{r['title']}\t{ctx}" + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--format", + choices=["grouped", "tsv"], + default="grouped", + help="output format (default: grouped text)", + ) + args = parser.parse_args() + + root = Path(__file__).resolve().parents[1] + rows = collect(root) + if args.format == "tsv": + print_tsv(rows) + else: + print_grouped(rows) + + +if __name__ == "__main__": + main() diff --git a/scripts/mcp-server.py b/scripts/mcp-server.py new file mode 100755 index 0000000..5b6d428 --- /dev/null +++ b/scripts/mcp-server.py @@ -0,0 +1,1010 @@ +#!/usr/bin/env python3 +"""MCP server for the knowledge-base repository. + +This server is the single interface an agent needs to work the vault over stdio +with JSON-RPC 2.0. It exposes graph and note *querying* (graph_summary, +graph_search_nodes, graph_neighbors, read_note, refresh_graph), *writing* +(write_note for text, capture_raw for verbatim bytes from a local file or an +http(s) download) and *publishing* (edit_note, git_commit, git_push), so an +agent without shell or filesystem tools can ingest a resource end to end. + +Every write is confined to the repository root, and staging is always explicit: +paths are named one by one, never `.` or `-A`. + +It auto-detects the transport layer: + - Content-Length framing (MCP spec, used by most clients) + - Newline-delimited JSON (used by Claude Code ≥ 2.1) +""" + +from __future__ import annotations + +import hashlib +import json +import shutil +import subprocess +import sys +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Dict, List + + +SERVER_NAME = "knowledge-base-mcp" +SERVER_VERSION = "0.1.0" + + +@dataclass +class AppState: + root: Path + graph_path: Path + + +class _StdinReader: + """Buffered stdin reader that supports transport auto-detection.""" + + def __init__(self, raw: Any) -> None: + self._raw = raw + self._prefix: bytes = b"" + + def read(self, n: int) -> bytes: + if self._prefix: + chunk, self._prefix = self._prefix[:n], self._prefix[n:] + remaining = n - len(chunk) + return chunk + (self._raw.read(remaining) if remaining else b"") + return self._raw.read(n) + + def readline(self) -> bytes: + if self._prefix: + nl = self._prefix.find(b"\n") + if nl != -1: + line, self._prefix = self._prefix[: nl + 1], self._prefix[nl + 1 :] + return line + prefix, self._prefix = self._prefix, b"" + return prefix + self._raw.readline() + return self._raw.readline() + + def peek_byte(self) -> bytes: + b = self._raw.read(1) + if b: + self._prefix = b + self._prefix + return b + + +_WHITESPACE_BYTES = {b"\r", b"\n", b"\t", b" "} + + +def _detect_transport(reader: _StdinReader) -> str: + """Peek at the first non-whitespace byte to decide 'ndjson' vs 'framed'. + + Leading ASCII whitespace bytes are discarded — they corrupt neither JSON + nor Content-Length headers, and some client wrappers emit stray newlines + before the first message. EOF before any non-whitespace defaults to + 'ndjson' so the read loop exits cleanly on an empty stream. + """ + while True: + b = reader._raw.read(1) + if not b: + return "ndjson" + if b not in _WHITESPACE_BYTES: + reader._prefix = b + reader._prefix + return "ndjson" if b == b"{" else "framed" + + +def _read_message_framed(reader: _StdinReader) -> Dict[str, Any] | None: + headers: Dict[str, str] = {} + while True: + line = reader.readline() + if not line: + return None + if line in (b"\r\n", b"\n"): + break + key, _, value = line.decode("utf-8").partition(":") + headers[key.strip().lower()] = value.strip() + + length_raw = headers.get("content-length") + if not length_raw: + raise ValueError("missing Content-Length header") + length = int(length_raw) + payload = reader.read(length) + if len(payload) != length: + raise ValueError("incomplete payload") + return json.loads(payload.decode("utf-8")) + + +def _read_message_ndjson(reader: _StdinReader) -> Dict[str, Any] | None: + while True: + line = reader.readline() + if not line: + return None + stripped = line.strip() + if not stripped: + continue + return json.loads(stripped.decode("utf-8")) + + +def _write_message_framed(stdout: Any, payload: Dict[str, Any]) -> None: + data = json.dumps(payload, ensure_ascii=True).encode("utf-8") + stdout.buffer.write(f"Content-Length: {len(data)}\r\n\r\n".encode("utf-8")) + stdout.buffer.write(data) + stdout.buffer.flush() + + +def _write_message_ndjson(stdout: Any, payload: Dict[str, Any]) -> None: + data = json.dumps(payload, ensure_ascii=True).encode("utf-8") + b"\n" + stdout.buffer.write(data) + stdout.buffer.flush() + + + +def _response(message_id: Any, result: Dict[str, Any]) -> Dict[str, Any]: + return {"jsonrpc": "2.0", "id": message_id, "result": result} + + +def _error(message_id: Any, code: int, message: str) -> Dict[str, Any]: + return { + "jsonrpc": "2.0", + "id": message_id, + "error": {"code": code, "message": message}, + } + + +def _load_graph(state: AppState) -> Dict[str, Any]: + if not state.graph_path.exists(): + raise FileNotFoundError( + f"graph not found at {state.graph_path}. Run scripts/generate-graph.py first." + ) + return json.loads(state.graph_path.read_text(encoding="utf-8")) + + +def _nodes_by_id(graph: Dict[str, Any]) -> Dict[str, Dict[str, Any]]: + nodes = graph.get("nodes", []) + return {str(node.get("id")): node for node in nodes if isinstance(node, dict)} + + +def _match_filters(node: Dict[str, Any], args: Dict[str, Any]) -> bool: + repo = args.get("repo") + note_type = args.get("type") + status = args.get("status") + tag = args.get("tag") + query = str(args.get("query", "")).strip().lower() + + if repo and node.get("repo") != repo: + return False + if note_type and node.get("type") != note_type: + return False + if status and node.get("status") != status: + return False + if tag and tag not in (node.get("tags") or []): + return False + if query: + haystacks = [ + str(node.get("id", "")).lower(), + str(node.get("title", "")).lower(), + str(node.get("path", "")).lower(), + " ".join(str(t).lower() for t in (node.get("tags") or [])), + ] + if not any(query in h for h in haystacks): + return False + return True + + +_WILDCARD_PATHS = {".", "./", "-A", "--all", "*", "-a", ":/"} + +# Ceiling for a single capture_raw call; papers and slide decks fit, a video +# download that would bloat the vault does not. +_DEFAULT_MAX_CAPTURE_BYTES = 100 * 1024 * 1024 +_DOWNLOAD_TIMEOUT_SECONDS = 60 + + +def _resolve_in_root(state: AppState, raw_path: Any, *, label: str = "path") -> Path: + """Turn a vault-relative path into an absolute path inside the repo root. + + Rejects empty values, absolute paths and anything escaping the root, so a + tool call can never touch a file outside the vault. + """ + value = str(raw_path or "").strip() + if not value: + raise ValueError(f"'{label}' is required") + candidate = Path(value) + if candidate.is_absolute(): + raise ValueError(f"'{label}' must be vault-relative, got absolute '{value}'") + root = state.root.resolve() + target = (root / candidate).resolve() + if target != root and not target.is_relative_to(root): + raise ValueError(f"'{label}' resolves outside repository root: '{value}'") + return target + + +def _relative_to_root(state: AppState, target: Path) -> str: + return target.relative_to(state.root.resolve()).as_posix() + + +def _run_git(state: AppState, args: List[str]) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", *args], + cwd=state.root, + check=False, + capture_output=True, + text=True, + ) + + +def _git_result(payload: Dict[str, Any], exit_code: int) -> Dict[str, Any]: + text = json.dumps(payload, indent=2) + if exit_code != 0: + return { + "isError": True, + "content": [{"type": "text", "text": text}], + "structuredContent": payload, + } + return _text_result(text, structured=payload) + + +def _tool_definitions() -> List[Dict[str, Any]]: + return [ + { + "name": "graph_summary", + "description": "Return aggregate counts from index/graph.json.", + "inputSchema": { + "type": "object", + "properties": { + "topLinked": {"type": "integer", "minimum": 1, "maximum": 50} + }, + }, + }, + { + "name": "graph_search_nodes", + "description": "Search graph nodes by metadata and free text.", + "inputSchema": { + "type": "object", + "properties": { + "repo": {"type": "string"}, + "type": {"type": "string"}, + "status": {"type": "string"}, + "tag": {"type": "string"}, + "query": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 200}, + }, + }, + }, + { + "name": "graph_neighbors", + "description": ( + "Show inbound and outbound links for a node ID. Each neighbor " + "includes the edge `type` (relationship, e.g. supersedes, " + "depends-on), `anchor_text` (the link's display text), and " + "`annotation` (what to expect in that note)." + ), + "inputSchema": { + "type": "object", + "required": ["id"], + "properties": { + "id": {"type": "string"}, + "limit": {"type": "integer", "minimum": 1, "maximum": 200}, + }, + }, + }, + { + "name": "read_note", + "description": ( + "Read note content by graph node ID or vault-relative path. " + "Provide exactly one of `id` or `path`. " + "If both or neither are provided, the call returns an error." + ), + "inputSchema": { + "type": "object", + "properties": { + "id": {"type": "string", "description": "Graph node ID (e.g. 'AGENTS')."}, + "path": {"type": "string", "description": "Vault-relative file path (e.g. 'README.md')."}, + }, + "additionalProperties": False, + }, + }, + { + "name": "refresh_graph", + "description": "Regenerate graph.json using scripts/generate-graph.py.", + "inputSchema": { + "type": "object", + "properties": { + "validate": {"type": "boolean"}, + }, + }, + }, + { + "name": "write_note", + "description": ( + "Create a note (or any text file) at a vault-relative path, " + "creating parent directories as needed. Fails if the file " + "exists unless `overwrite` is true. The generated " + "`index/graph.json` is off limits — use `refresh_graph`." + ), + "inputSchema": { + "type": "object", + "required": ["path", "content"], + "properties": { + "path": { + "type": "string", + "description": "Vault-relative path, e.g. '03_resources/demo/index.md'.", + }, + "content": {"type": "string", "description": "Full file content."}, + "overwrite": { + "type": "boolean", + "description": "Replace an existing file (default false).", + }, + }, + "additionalProperties": False, + }, + }, + { + "name": "edit_note", + "description": ( + "Replace an exact string inside an existing note. `old_string` " + "must occur exactly once unless `replace_all` is true." + ), + "inputSchema": { + "type": "object", + "required": ["path", "old_string", "new_string"], + "properties": { + "path": {"type": "string", "description": "Vault-relative path of the note."}, + "old_string": {"type": "string", "description": "Text to replace, matched verbatim."}, + "new_string": {"type": "string", "description": "Replacement text."}, + "replace_all": { + "type": "boolean", + "description": "Replace every occurrence (default false).", + }, + }, + "additionalProperties": False, + }, + }, + { + "name": "capture_raw", + "description": ( + "Copy a local file or download an http(s) URL into the vault " + "byte-for-byte — the way an original PDF, image, or archive is " + "preserved under `03_resources//raw/`. Provide exactly one " + "of `source_path` (absolute or CWD-relative path on this machine) " + "or `url`. Binary-safe; returns size and sha256. Read the " + "captured file back with `read_note`, which reports binary files " + "with an absolute path to open with a file-reading tool." + ), + "inputSchema": { + "type": "object", + "required": ["path"], + "properties": { + "path": { + "type": "string", + "description": "Vault-relative destination, e.g. '03_resources/demo/raw/paper.pdf'.", + }, + "source_path": { + "type": "string", + "description": "Path of the local file to copy.", + }, + "url": { + "type": "string", + "description": "http(s) URL to download verbatim.", + }, + "overwrite": { + "type": "boolean", + "description": "Replace an existing capture (default false).", + }, + "max_bytes": { + "type": "integer", + "minimum": 1, + "description": "Reject captures larger than this (default 100 MiB).", + }, + }, + "additionalProperties": False, + }, + }, + { + "name": "git_commit", + "description": ( + "Stage the named vault-relative paths individually and commit " + "them. `paths` must list every path explicitly — wildcard " + "staging ('.', '-A', '*') is rejected. Returns an error result " + "if git refuses the commit (e.g. nothing staged)." + ), + "inputSchema": { + "type": "object", + "required": ["message", "paths"], + "properties": { + "message": {"type": "string", "description": "Commit message (Conventional Commits)."}, + "paths": { + "type": "array", + "minItems": 1, + "items": {"type": "string"}, + "description": "Vault-relative paths to stage, one entry per file or directory.", + }, + }, + "additionalProperties": False, + }, + }, + { + "name": "git_push", + "description": ( + "Push commits to a remote. Defaults to 'origin' and the current " + "branch. Returns an error result if git refuses the push." + ), + "inputSchema": { + "type": "object", + "properties": { + "remote": {"type": "string", "description": "Remote name (default 'origin')."}, + "branch": {"type": "string", "description": "Branch to push (default: current branch)."}, + "set_upstream": { + "type": "boolean", + "description": "Pass --set-upstream (default false).", + }, + }, + "additionalProperties": False, + }, + }, + ] + + +def _text_result(text: str, structured: Dict[str, Any] | None = None) -> Dict[str, Any]: + result: Dict[str, Any] = {"content": [{"type": "text", "text": text}]} + if structured is not None: + result["structuredContent"] = structured + return result + + +def _tool_graph_summary(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + graph = _load_graph(state) + nodes = graph.get("nodes", []) + edges = graph.get("edges", []) + top_n = int(args.get("topLinked", 10)) + + by_repo: Dict[str, int] = {} + by_type: Dict[str, int] = {} + by_status: Dict[str, int] = {} + for node in nodes: + by_repo[str(node.get("repo", "unknown"))] = ( + by_repo.get(str(node.get("repo", "unknown")), 0) + 1 + ) + by_type[str(node.get("type", "unknown"))] = ( + by_type.get(str(node.get("type", "unknown")), 0) + 1 + ) + by_status[str(node.get("status", "unknown"))] = ( + by_status.get(str(node.get("status", "unknown")), 0) + 1 + ) + + sorted_nodes = sorted( + nodes, + key=lambda n: int(n.get("inbound_count", 0)), + reverse=True, + )[:top_n] + payload = { + "version": graph.get("version"), + "generated_at": graph.get("generated_at"), + "node_count": len(nodes), + "edge_count": len(edges), + "by_repo": by_repo, + "by_type": by_type, + "by_status": by_status, + "top_linked": [ + { + "id": node.get("id"), + "title": node.get("title"), + "path": node.get("path"), + "inbound_count": node.get("inbound_count", 0), + } + for node in sorted_nodes + ], + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_graph_search_nodes(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + graph = _load_graph(state) + nodes = graph.get("nodes", []) + limit = int(args.get("limit", 30)) + filtered = [node for node in nodes if _match_filters(node, args)] + filtered.sort(key=lambda n: str(n.get("id", ""))) + sliced = filtered[:limit] + payload = { + "total_matches": len(filtered), + "returned": len(sliced), + "items": sliced, + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_graph_neighbors(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + graph = _load_graph(state) + node_id = str(args["id"]) + limit = int(args.get("limit", 50)) + nodes = _nodes_by_id(graph) + if node_id not in nodes: + raise ValueError(f"node '{node_id}' not found") + + edges = graph.get("edges", []) + outbound_edges = [e for e in edges if e.get("from") == node_id] + inbound_edges = [e for e in edges if e.get("to") == node_id] + + def describe(edge_list: List[Dict[str, Any]], direction: str) -> List[Dict[str, Any]]: + # neighbor id is the "other end" relative to node_id + other_key = "to" if direction == "outbound" else "from" + result: List[Dict[str, Any]] = [] + for edge in edge_list: + item_id = str(edge.get(other_key)) + node = nodes.get(item_id) + entry = { + "id": node.get("id") if node else item_id, + "title": node.get("title") if node else "", + "path": node.get("path") if node else "", + "direction": direction, + "type": edge.get("type", "related"), + "anchor_text": edge.get("anchor_text", ""), + "annotation": edge.get("annotation", ""), + } + result.append(entry) + return result[:limit] + + payload = { + "node": nodes[node_id], + "inbound_count": len(inbound_edges), + "outbound_count": len(outbound_edges), + "inbound": describe(inbound_edges, "inbound"), + "outbound": describe(outbound_edges, "outbound"), + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_read_note(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + has_id = bool(args.get("id")) + has_path = bool(args.get("path")) + + if has_id and has_path: + raise ValueError("provide exactly one of 'id' or 'path', not both") + if not has_id and not has_path: + raise ValueError("exactly one of 'id' or 'path' is required") + + path_value = args.get("path") + + if has_id: + nodes = _nodes_by_id(_load_graph(state)) + node = nodes.get(str(args["id"])) + if not node: + raise ValueError(f"node '{args['id']}' not found") + path_value = node.get("path") + + target = (state.root / str(path_value)).resolve() + if not target.is_relative_to(state.root.resolve()): + raise ValueError("note path resolves outside repository root") + if not target.exists(): + raise FileNotFoundError(f"note not found at {target}") + + relative = str(target.relative_to(state.root).as_posix()) + data = target.read_bytes() + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + payload = { + "path": relative, + "binary": True, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + "hint": ( + f"{target} is not UTF-8 text (e.g. a PDF or image captured under " + "raw/). Open it with a file-reading tool that handles binaries; " + "this server returns text only." + ), + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + payload = { + "path": relative, + "binary": False, + "content": text, + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_refresh_graph(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + command = [sys.executable, "scripts/generate-graph.py"] + if bool(args.get("validate", False)): + command.append("--validate") + + proc = subprocess.run( + command, + cwd=state.root, + check=False, + capture_output=True, + text=True, + ) + + payload = { + "command": command, + "exit_code": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + } + if proc.returncode != 0: + return { + "isError": True, + "content": [{"type": "text", "text": json.dumps(payload, indent=2)}], + "structuredContent": payload, + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_write_note(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + if "content" not in args: + raise ValueError("'content' is required") + content = str(args["content"]) + target = _resolve_in_root(state, args.get("path")) + relative = _relative_to_root(state, target) + if relative == state.graph_path.relative_to(state.root.resolve()).as_posix(): + raise ValueError( + "index/graph.json is generated \u2014 use refresh_graph instead of writing it" + ) + + existed = target.exists() + if existed and not bool(args.get("overwrite", False)): + raise FileExistsError( + f"note already exists at {relative}; pass overwrite=true to replace it" + ) + if existed and target.is_dir(): + raise ValueError(f"'{relative}' is a directory") + + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(content, encoding="utf-8") + payload = { + "path": relative, + "created": not existed, + "bytes": len(content.encode("utf-8")), + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_edit_note(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + if "old_string" not in args or "new_string" not in args: + raise ValueError("'old_string' and 'new_string' are required") + old_string = str(args["old_string"]) + new_string = str(args["new_string"]) + if not old_string: + raise ValueError("'old_string' must not be empty") + if old_string == new_string: + raise ValueError("'old_string' and 'new_string' are identical") + + target = _resolve_in_root(state, args.get("path")) + relative = _relative_to_root(state, target) + if not target.is_file(): + raise FileNotFoundError(f"note not found at {relative}") + + text = target.read_text(encoding="utf-8") + occurrences = text.count(old_string) + if occurrences == 0: + raise ValueError(f"'old_string' not found in {relative}") + replace_all = bool(args.get("replace_all", False)) + if occurrences > 1 and not replace_all: + raise ValueError( + f"'old_string' occurs {occurrences} times in {relative}; " + "make it unique or pass replace_all=true" + ) + + updated = text.replace(old_string, new_string) + target.write_text(updated, encoding="utf-8") + payload = { + "path": relative, + "replacements": occurrences if replace_all else 1, + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _download_bytes(url: str, max_bytes: int) -> bytes: + """Fetch an http(s) URL into memory, refusing anything over `max_bytes`.""" + request = urllib.request.Request( + url, headers={"User-Agent": f"{SERVER_NAME}/{SERVER_VERSION}"} + ) + with urllib.request.urlopen(request, timeout=_DOWNLOAD_TIMEOUT_SECONDS) as response: + declared = response.headers.get("Content-Length") + if declared and declared.isdigit() and int(declared) > max_bytes: + raise ValueError( + f"remote file is {declared} bytes, over the {max_bytes} byte limit" + ) + data = response.read(max_bytes + 1) + if len(data) > max_bytes: + raise ValueError(f"download exceeds the {max_bytes} byte limit") + return data + + +def _tool_capture_raw(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + source_path = str(args.get("source_path") or "").strip() + url = str(args.get("url") or "").strip() + if bool(source_path) == bool(url): + raise ValueError("provide exactly one of 'source_path' or 'url'") + + max_bytes = int(args.get("max_bytes", _DEFAULT_MAX_CAPTURE_BYTES)) + if max_bytes < 1: + raise ValueError("'max_bytes' must be positive") + + target = _resolve_in_root(state, args.get("path")) + relative = _relative_to_root(state, target) + existed = target.exists() + if existed and not bool(args.get("overwrite", False)): + raise FileExistsError( + f"capture already exists at {relative}; pass overwrite=true to replace it" + ) + if existed and target.is_dir(): + raise ValueError(f"'{relative}' is a directory") + + if source_path: + origin = Path(source_path).expanduser() + if not origin.exists(): + raise FileNotFoundError(f"source file not found at {origin}") + if not origin.is_file(): + raise ValueError(f"source is not a file: {origin}") + size = origin.stat().st_size + if size > max_bytes: + raise ValueError( + f"source file is {size} bytes, over the {max_bytes} byte limit" + ) + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(origin, target) + data = target.read_bytes() + source_label = str(origin) + else: + scheme = url.split(":", 1)[0].lower() + if scheme not in ("http", "https"): + raise ValueError(f"only http(s) URLs can be captured, got '{url}'") + try: + data = _download_bytes(url, max_bytes) + except (urllib.error.URLError, urllib.error.HTTPError, OSError) as exc: + payload = { + "url": url, + "path": relative, + "exit_code": 1, + "stderr": f"download failed: {exc}", + } + return _git_result(payload, 1) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(data) + source_label = url + + payload = { + "path": relative, + "absolute_path": str(target), + "source": source_label, + "created": not existed, + "bytes": len(data), + "sha256": hashlib.sha256(data).hexdigest(), + } + return _text_result(json.dumps(payload, indent=2), structured=payload) + + +def _tool_git_commit(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + message = str(args.get("message", "")).strip() + if not message: + raise ValueError("'message' is required") + + raw_paths = args.get("paths") + if not isinstance(raw_paths, list) or not raw_paths: + raise ValueError("'paths' must list at least one vault-relative path") + + relatives: List[str] = [] + for raw in raw_paths: + value = str(raw or "").strip() + if value in _WILDCARD_PATHS or value.startswith("-"): + raise ValueError( + f"wildcard staging is not allowed: '{value}'. Name each path explicitly." + ) + target = _resolve_in_root(state, value, label="paths[]") + relative = _relative_to_root(state, target) + if not target.exists(): + raise FileNotFoundError(f"path not found at {relative}") + relatives.append(relative) + + relatives = sorted(set(relatives)) + for relative in relatives: + add = _run_git(state, ["add", "--", relative]) + if add.returncode != 0: + payload = { + "command": ["git", "add", "--", relative], + "exit_code": add.returncode, + "stdout": add.stdout, + "stderr": add.stderr, + "committed_paths": [], + } + return _git_result(payload, add.returncode) + + commit = _run_git(state, ["commit", "-m", message]) + payload = { + "command": ["git", "commit", "-m", message], + "exit_code": commit.returncode, + "stdout": commit.stdout, + "stderr": commit.stderr, + "committed_paths": relatives if commit.returncode == 0 else [], + } + if commit.returncode == 0: + head = _run_git(state, ["rev-parse", "--short", "HEAD"]) + payload["commit"] = head.stdout.strip() + return _git_result(payload, commit.returncode) + + +def _tool_git_push(state: AppState, args: Dict[str, Any]) -> Dict[str, Any]: + remote = str(args.get("remote") or "origin").strip() + branch = str(args.get("branch") or "").strip() + for label, value in (("remote", remote), ("branch", branch)): + if value.startswith("-"): + raise ValueError(f"'{label}' must not look like an option: '{value}'") + if not remote: + raise ValueError("'remote' must not be empty") + + if not branch: + head = _run_git(state, ["rev-parse", "--abbrev-ref", "HEAD"]) + if head.returncode != 0: + payload = { + "command": ["git", "rev-parse", "--abbrev-ref", "HEAD"], + "exit_code": head.returncode, + "stdout": head.stdout, + "stderr": head.stderr, + } + return _git_result(payload, head.returncode) + branch = head.stdout.strip() + + command = ["push"] + if bool(args.get("set_upstream", False)): + command.append("--set-upstream") + command += [remote, branch] + + push = _run_git(state, command) + payload = { + "command": ["git", *command], + "remote": remote, + "branch": branch, + "exit_code": push.returncode, + "stdout": push.stdout, + "stderr": push.stderr, + } + return _git_result(payload, push.returncode) + + +def _call_tool(state: AppState, name: str, arguments: Dict[str, Any]) -> Dict[str, Any]: + if name == "graph_summary": + return _tool_graph_summary(state, arguments) + if name == "graph_search_nodes": + return _tool_graph_search_nodes(state, arguments) + if name == "graph_neighbors": + return _tool_graph_neighbors(state, arguments) + if name == "read_note": + return _tool_read_note(state, arguments) + if name == "refresh_graph": + return _tool_refresh_graph(state, arguments) + if name == "write_note": + return _tool_write_note(state, arguments) + if name == "edit_note": + return _tool_edit_note(state, arguments) + if name == "capture_raw": + return _tool_capture_raw(state, arguments) + if name == "git_commit": + return _tool_git_commit(state, arguments) + if name == "git_push": + return _tool_git_push(state, arguments) + raise ValueError(f"unknown tool '{name}'") + + +def _list_resources(state: AppState) -> Dict[str, Any]: + return { + "resources": [ + { + "uri": "repo://graph", + "name": "Repository Graph", + "description": "Raw index/graph.json", + "mimeType": "application/json", + }, + { + "uri": "repo://readme", + "name": "Repository README", + "description": "Root README.md", + "mimeType": "text/markdown", + }, + ] + } + + +def _read_resource(state: AppState, uri: str) -> Dict[str, Any]: + if uri == "repo://graph": + text = state.graph_path.read_text(encoding="utf-8") + return { + "contents": [ + { + "uri": uri, + "mimeType": "application/json", + "text": text, + } + ] + } + if uri == "repo://readme": + readme = (state.root / "README.md").read_text(encoding="utf-8") + return {"contents": [{"uri": uri, "mimeType": "text/markdown", "text": readme}]} + raise ValueError(f"unknown resource '{uri}'") + + +def _handle_request(state: AppState, message: Dict[str, Any]) -> Dict[str, Any] | None: + method = message.get("method") + message_id = message.get("id") + params = message.get("params") or {} + + if method == "notifications/initialized": + return None + + if method == "initialize": + result = { + "protocolVersion": "2024-11-05", + "serverInfo": {"name": SERVER_NAME, "version": SERVER_VERSION}, + "capabilities": {"tools": {}, "resources": {}}, + } + return _response(message_id, result) + + if method == "tools/list": + return _response(message_id, {"tools": _tool_definitions()}) + + if method == "tools/call": + try: + tool_name = str(params.get("name", "")) + arguments = params.get("arguments") or {} + if not isinstance(arguments, dict): + raise ValueError("tool arguments must be an object") + result = _call_tool(state, tool_name, arguments) + return _response(message_id, result) + except Exception as exc: + error_result = { + "isError": True, + "content": [{"type": "text", "text": str(exc)}], + } + return _response(message_id, error_result) + + if method == "resources/list": + return _response(message_id, _list_resources(state)) + + if method == "resources/read": + try: + uri = str(params.get("uri", "")) + return _response(message_id, _read_resource(state, uri)) + except Exception as exc: + return _error(message_id, -32602, str(exc)) + + if message_id is None: + return None + return _error(message_id, -32601, f"method not found: {method}") + + +def main() -> None: + root = Path(__file__).resolve().parents[1] + state = AppState(root=root, graph_path=root / "index" / "graph.json") + + reader = _StdinReader(sys.stdin.buffer) + try: + transport = _detect_transport(reader) + except OSError as exc: + print( + json.dumps({"error": "transport detection failed", "detail": str(exc)}), + file=sys.stderr, + ) + sys.exit(1) + if transport == "ndjson": + read_msg = _read_message_ndjson + write_msg = _write_message_ndjson + else: + read_msg = _read_message_framed + write_msg = _write_message_framed + + while True: + try: + message = read_msg(reader) + except Exception as exc: + write_msg(sys.stdout, _error(None, -32700, f"parse error: {exc}")) + continue + + if message is None: + break + + response = _handle_request(state, message) + if response is not None: + write_msg(sys.stdout, response) + + +if __name__ == "__main__": + main() diff --git a/scripts/pre-commit b/scripts/pre-commit new file mode 100755 index 0000000..58a0ca4 --- /dev/null +++ b/scripts/pre-commit @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -euo pipefail + +echo "[pre-commit] Generating and validating graph..." +make graph + +if ! git diff --quiet -- index/graph.json; then + echo "[pre-commit] index/graph.json was modified by generation." + echo "[pre-commit] Run 'make graph' and stage index/graph.json before committing." + exit 1 +fi diff --git a/skills/README.md b/skills/README.md new file mode 100644 index 0000000..88cee81 --- /dev/null +++ b/skills/README.md @@ -0,0 +1 @@ +When creating a new skill or renaming the folder, please adjust the links in `./.claude/skills/` diff --git a/skills/knowledge-base-ingest-resource/SKILL.md b/skills/knowledge-base-ingest-resource/SKILL.md new file mode 100644 index 0000000..3b682e2 --- /dev/null +++ b/skills/knowledge-base-ingest-resource/SKILL.md @@ -0,0 +1,284 @@ +--- +name: knowledge-base-ingest-resource +description: Ingest an external resource (local file, URL/webpage, PDF, YouTube video, etc.) into this knowledge-base vault (PARA-organized). Copies the original into 03_resources//raw/, reads the content, breaks it into individual topics, and wires each topic to related notes already in the vault (internal typed markdown links) and to outside sources (URLs). For YouTube, pulls the transcript into raw/. Use when the user says "ingest", "add this resource", "save this link/video/paper", or hands you a file/URL to capture and link. +--- + +# Ingest a Resource + +Capture an external resource into the vault as a reference note, store its +original bytes under `raw/`, decompose it into topics, and link those topics into +the knowledge graph. Follow every step. + +Obey `AGENTS.md` and `02_areas/ways-of-working.md` (read them if not already +loaded this session; if `ways-of-working.md` does not exist yet, skip it). + +The output is a `type: note` reference note at +`03_resources//index.md`, one `type: note` **topic note** per +distinct topic at `03_resources//.md`, and the +untouched original preserved in `03_resources//raw/` — all linked +into the graph with typed markdown links (`[title](id){"type":"…","link-info":"…"}`). + +``` +03_resources// + index.md # the resource note (summary, source, topic index) + .md # one note per topic distilled from the resource + .md + raw/ # original bytes / transcript / fetched page +``` + +## 1. Identify the resource and its type + +The resource comes from the conversation — a file path, a URL, a pasted link, or +an attachment. Classify it; this drives how you capture it in step 3: + +| Type | Signal | Capture into `raw/` | +|-----------------|-----------------------------------------------------|----------------------------------------------| +| Local file | a path that exists on disk | copy the file verbatim | +| YouTube video | host is youtube.com / youtu.be | pull the **transcript** (+ metadata) | +| Webpage / link | any other http(s) URL | save the fetched page (markdown/html) + URL | +| PDF / doc | path or URL ending in .pdf/.docx/... | copy/download the binary verbatim | + +If the resource type is genuinely ambiguous (e.g. a bare string that could be a +slug or a search term), ask the user. Otherwise proceed. + +## 2. Derive the resource name and create the folder + +Pick a short kebab-case `` from the title / filename / video +title. Keep it descriptive and unique (e.g. `cyclonedx-sbom-spec`, +`oauth2-rfc6749`). + +```bash +ls 03_resources/ # check the slug is not already taken +mkdir -p 03_resources//raw +``` + +If the slug already exists, either you are re-ingesting (update in place, tell the +user) or pick a more specific slug. + +## 3. Copy the original into `raw/` + +Always preserve the source bytes untouched. Pick the branch that matches step 1. + +**Local file** — copy verbatim, keep the original name: +```bash +cp "" "03_resources//raw/" +``` + +**PDF / binary at a URL** — download verbatim: +```bash +curl -fsSL "" -o "03_resources//raw/" +``` + +**Webpage / link** — record the URL and save the readable content. Use `WebFetch` +to retrieve it (request markdown), then write the result and a `source.url`: +```bash +printf '%s\n' "" > "03_resources//raw/source.url" +``` +Save the fetched body to `raw/page.md` (write the `WebFetch` output to that file). + +**YouTube video** — pull the transcript into `raw/`. Record the URL first, then +try, in order: +```bash +printf '%s\n' "" > "03_resources//raw/source.url" + +# Preferred: yt-dlp grabs subs + metadata without downloading the video +yt-dlp --skip-download --write-subs --write-auto-subs --sub-lang en \ + --convert-subs srt --write-info-json \ + -o "03_resources//raw/%(title)s.%(ext)s" "" + +# Fallback if yt-dlp is absent: python transcript API +python3 -c "import youtube_transcript_api" 2>/dev/null && \ + python3 -c "from youtube_transcript_api import YouTubeTranscriptApi as A; \ +import sys; vid=''; \ +open('03_resources//raw/transcript.txt','w').write(\ +'\n'.join(s['text'] for s in A.get_transcript(vid)))" +``` +If neither tool is installed, tell the user which to install (`pip install yt-dlp` +or `pip install youtube-transcript-api`) and stop before fabricating a transcript. +Never invent transcript text. + +## 4. Read the captured content + +Read what you just stored in `raw/` (the file, `page.md`, the transcript). This — +not the live URL — is the source of truth for the rest of the steps, so the note +matches the bytes we preserved. For long content, read enough to identify every +distinct topic, not just the intro. + +## 5. Decompose into topics and find links + +Reason about the **individual topics** inside the resource — the concrete +concepts, standards, formats, tools, or entities it discusses. For each topic, +decide whether the vault already knows about it, and whether it points outward. + +For every candidate topic, search the vault for a matching note: + +```bash +# free-text over the graph (id / title / path / tags) +jq -r --arg q "" '.nodes[] + | select((.id|ascii_downcase|contains($q)) or (.title|ascii_downcase|contains($q))) + | "\(.id)\t\(.path)"' index/graph.json + +# by tag +jq -r --arg t "" '.nodes[] | select(.tags[]? == $t) | "\(.id)\t\(.path)"' index/graph.json + +# full-text fallback across the PARA buckets +grep -rIl "" 01_projects 02_areas 03_resources 04_archives +``` + +Prefer the `knowledge-database` MCP server when available (`graph_search_nodes`, +`graph_neighbors`, `read_note`) — it is the same index. Open promising matches and +confirm they are really the same concept before linking. + +Classify each topic: +- **Internal** — the vault has a note for it → link with + `[](<note-id>){"type":"…"}`. +- **External** — no vault note, but it has an authoritative source (a spec, RFC, + vendor page) → record the URL so it can become a note later. +- **Unlinked** — neither yet → list it as a candidate so a future note can fill it. + +> Example: a CycloneDX SBOM resource that mentions CPE and purls → search the +> vault for `cpe`, `purl`, `sbom`; link `[text](id){"type":"…"}` to whichever +> notes exist, and +> capture the CPE/purl spec URLs as external for the rest. A resource describing +> CycloneDX **vendor references** → reason about which existing supplier/vendor or +> component-identity notes it relates to and link those. + +## 6. Write a note per topic + +Each distinct topic from step 5 becomes its own note at +`03_resources/<resource-name>/<topic-name>.md` (kebab-case slug). This is what +makes topics first-class, individually linkable graph nodes. + +```md +--- +id: resource/<resource-name>/<topic-name> +title: <Topic Title> +type: note +status: active +tags: [resource, <topic-name>, <extra-tags...>] +source: <original url or path> +ingested: <today, YYYY-MM-DD> +last_reviewed: <today, YYYY-MM-DD> +--- + +# <Topic Title> + +<What the resource says about this topic — a few sentences, in your words.> + +## In this resource +From [<resource title>](resource/<resource-name>){"type":"part-of","link-info":"<how the topic appears / why it matters here>"} + +## Related +- [<title>](<existing-vault-note-id>){"type":"related","link-info":"<what to expect there>"} # internal +- <https://authoritative-spec-url> # external — spec / RFC / vendor page +``` + +Links are **markdown links** whose target is the note `id`, with an optional +`{"type":"…","link-info":"…"}` brace. `type` must be one of the controlled +vocabulary (the pre-commit hook rejects others): +`related, supersedes, superseded-by, depends-on, part-of, parent, derived-from, +see-also, source`. Omit the brace for a plain `related` link. `link-info` is what +the *source* needs from the *destination* — "what to expect there". + +- One topic = one file. Skip trivia; write a note only for topics worth recalling. +- If the vault **already** has a note for the topic (an internal match from step + 5), do NOT duplicate it — link it from the resource note instead and, if useful, + add a backlink (step 7). Create a topic note here only for topics the vault does + not yet cover. +- Link each topic note back to its resource with + `[<resource title>](resource/<resource-name>){"type":"part-of"}`, and outward to + internal notes (same link form) and external sources (URLs). +- Legacy `[[wiki links]]` are rejected by `make validate-graph` — never write them. + +## 7. Write the resource note (index) + +Filename: `03_resources/<resource-name>/index.md`. This is the hub that ties the +raw source to the topic notes. + +```md +--- +id: resource/<resource-name> +title: <Human Title> +type: note +status: active +tags: [resource, <topic-tags...>] +source: <original url or path> +ingested: <today, YYYY-MM-DD> +last_reviewed: <today, YYYY-MM-DD> +--- + +# <Human Title> + +## Summary +<2–5 sentences: what this resource is and why it is worth keeping.> + +## Source +- Type: <local file | webpage | youtube | pdf | ...> +- Origin: <url or original path> +- Raw: `03_resources/<resource-name>/raw/` + +## Topics +- [<Topic>](resource/<resource-name>/<topic-name>){"type":"part-of","link-info":"<one line>"} # new topic note +- [<Topic>](<existing-note-id>){"type":"related","link-info":"<one line>"} # vault already had it +- **<Topic>** — <one line> → <https://external-spec-url> # external only +- **<Topic>** — <one line> (no note yet) # candidate + +## Related +- [<title>](<related-note-id>){"type":"related","link-info":"<what to expect there>"} +``` + +- `ingested` and `last_reviewed` = today (the `currentDate` from session context). +- `type` must stay within the vault taxonomy enum + (`hub|project|guideline|policy|feature|roadmap|decision|note|meta`) — use `note`; + the `resource` tag is what marks it. +- Markdown link targets resolve by `id`, rel-path, title, or unique stem — always + use the `id` as the target, e.g. `[Human title](resource/foo/bar)`. +- Do not leave a section empty; write `None.` if a section truly has nothing. + +## 8. Wire backlinks + +So the resource is reachable in the graph, not just reaching out: + +- Add a `[<title>](resource/<resource-name>)` (or a relevant topic note) link from + the most relevant existing notes (the strong internal topics from step 5) — e.g. + under a `## Resources` or `## Related` section, creating it if absent. Add a + `{"type":"…","link-info":"…"}` brace where the relationship is more specific than + `related`. Use individual edits. +- If several resources accumulate, maintain `03_resources/index.md` as a hub that + lists them, and link the new note from there (create the hub if it does not + exist, with `type: hub`, `tags: [resource, hub]`). + +## 9. Regenerate and validate the graph + +```bash +make graph +``` + +This regenerates `index/graph.json` and validates links and edge relationship +types. If validation reports unresolved links, fix the targets (typos, wrong `id`, +or a link to a note that does not exist yet — either create it or downgrade it to +plain text / external URL); if it reports an unknown edge `type`, correct it to a +value in the controlled vocabulary. Re-run until clean. The pre-commit hook rejects +a stale or +invalid graph, so this must pass. + +## 10. Stage (do not commit unless asked) + +Per `ways-of-working.md`, NEVER use `git add -A` or `git add .`. Stage each +touched path individually: + +```bash +git add 03_resources/<resource-name>/ # note + raw/ +git add index/graph.json +git add <any-existing-note-you-backlinked>.md +``` + +Only commit or open a PR if the user explicitly asks. Changes go via small, clear +pull requests. + +## 11. Report + +Summarize: the resource id/path, type, what landed in `raw/`, the topic notes you +created (`<topic-name>.md`), which topics were linked to existing vault notes +(typed markdown links) vs. captured as external vs. left as candidates, files touched, and the +graph-validation result. diff --git a/skills/knowledge-base-read/SKILL.md b/skills/knowledge-base-read/SKILL.md new file mode 100644 index 0000000..6bf880a --- /dev/null +++ b/skills/knowledge-base-read/SKILL.md @@ -0,0 +1,108 @@ +--- +name: knowledge-base-read +description: Read and query the knowledge-base vault — a personal docs-as-code knowledge database (PARA-organized). Use when the user asks what is known about a repo, policy, decision (ADR), area, or project; wants context before working in a repo; or asks to find/navigate notes by tag, type, repo, or link. Resolves the question against the Markdown notes and the generated graph index. +--- + +# Read the Knowledge Base + +This vault is a personal docs-as-code knowledge database, +organized with the [PARA method](https://fortelabs.com/blog/para/). +Markdown is the source of truth; `index/graph.json` is a generated navigation +index. Use this skill to answer questions FROM the vault and to surface the +exact notes that back the answer. + +Obey `AGENTS.md` (read them if not loaded this +session). This skill is read-only — do not modify notes here. + +## Layout (PARA) + +``` +01_projects/ active efforts with a goal and an end +02_areas/ ongoing standards: policies, value-stream areas, decisions +02_areas/decisions/ global ADRs +03_resources/meta/ taxonomy + note templates +03_resources/repos/<repo>/ per-repo info.md (+ decisions/) +03_resources/repos/repo-landscape.md index of documented repos +04_archives/ inactive items +index/graph.json generated graph (nodes + edges) +README.md vault hub; AGENT_QUERY_GUIDE.md graph query reference +``` + +Every note carries taxonomy frontmatter: `id`, `title`, `type` +(`hub|project|guideline|policy|feature|roadmap|decision|note|meta`), `status` +(`active|draft|deprecated`), `tags`, `last_reviewed`, optional `repo`. Notes are +linked with typed markdown links — `[text](note-id){"type":"…"}` — that resolve +by `id`. Legacy `[[wiki links]]` are rejected by `make validate-graph`. + +## 1. Orient + +Start broad, then narrow: + +- `README.md` — vault hub and entry points. +- `03_resources/repos/repo-landscape.md` — index of every documented repo, grouped. +- `02_areas/ways-of-working.md` — global engineering policy. +- `03_resources/meta/taxonomy.md` — what the frontmatter fields and PARA buckets mean. + +For a repo question, read `03_resources/repos/<repo>/info.md` directly +(`ls 03_resources/repos/` to confirm the repo is documented; if absent, say so — +it is not yet in the vault). + +## 2. Query the graph + +`index/graph.json` is for navigation by tag / type / repo / links. Use `jq`. +Full schema and more examples are in `AGENT_QUERY_GUIDE.md`. + +```bash +# all documented repos +jq -r '.nodes[].repo' index/graph.json | sort -u + +# every note in one repo +jq -r '.nodes[] | select(.repo=="tenant-controller") | "\(.id)\t\(.path)"' index/graph.json + +# all notes of a type (e.g. policies, decisions, projects) +jq -r '.nodes[] | select(.type=="policy") | "\(.id)\t\(.path)"' index/graph.json + +# notes carrying a tag +jq -r '.nodes[] | select(.tags[]? == "kcp") | "\(.id)\t\(.path)"' index/graph.json + +# most-linked (central) notes +jq -r '.nodes | sort_by(-(.inbound_count // 0))[] | "\(.inbound_count // 0)\t\(.id)"' index/graph.json | head + +# outgoing links from a note, with relationship type + annotation +NOTE_ID="repo/controller-util/info" +jq -r --arg id "$NOTE_ID" '.edges[] | select(.from==$id) | "\(.type)\t\(.to)\t\(.annotation)"' index/graph.json + +# incoming links to a note (who references it, and how) +jq -r --arg id "$NOTE_ID" '.edges[] | select(.to==$id) | "\(.type)\t\(.from)\t\(.annotation)"' index/graph.json + +# edges of a specific relationship type (e.g. supersedes, depends-on) +jq -r '.edges[] | select(.type=="supersedes") | "\(.from) -> \(.to)"' index/graph.json +``` + +Full-text fallback when you do not know the tag/type/id: + +```bash +grep -rIl "<term>" 01_projects 02_areas 03_resources 04_archives +``` + +## 3. Read the source notes + +The graph gives ids and paths; the ANSWER lives in the Markdown. Open the +matched `.path` files and read them. Follow the typed markdown links +`[text](id){"type":"…"}`, resolving each to its node `path` to pull in related +policies, decisions, and repo info. Use each edge's `type` and `annotation` to +judge which links are worth following; links are the intended navigation, so +traverse them rather than guessing. + +If the graph looks stale or a path is missing, you may regenerate read-only: +`python3 scripts/generate-graph.py` (does not touch Markdown). Do not commit. + +## 4. Answer with citations + +- Answer the question directly from what the notes say. +- Cite the backing notes by vault-relative `path` (e.g. + `03_resources/repos/gittea/info.md`) and/or `id` so the user can verify. +- Distinguish vault facts from inference. If the vault does not cover it, say so + plainly — do not invent. Note `status: draft`/`deprecated` when relevant. +- For "what links to / relates to X" questions, report the inbound/outbound + edges you found, including each edge's relationship `type` and `annotation`. diff --git a/tests/test_mcp_server.py b/tests/test_mcp_server.py new file mode 100644 index 0000000..cb99e8c --- /dev/null +++ b/tests/test_mcp_server.py @@ -0,0 +1,432 @@ +#!/usr/bin/env python3 +"""Tests for the knowledge-base MCP server's write and git tools. + +Covers the tools an ingesting agent needs when the MCP server is its only way +into the vault: creating notes, editing notes, capturing raw bytes (local files +and downloads), committing staged paths, and pushing. Each test drives the +server through `_call_tool` against a throwaway repository so no test touches +the real vault. +""" + +from __future__ import annotations + +import hashlib +import http.server +import json +import subprocess +import sys +import tempfile +import threading +import unittest +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import importlib + +mcp = importlib.import_module("mcp-server") + + +def _git(cwd: Path, *args: str) -> subprocess.CompletedProcess: + return subprocess.run( + ["git", *args], cwd=cwd, check=True, capture_output=True, text=True + ) + + +class ToolTestCase(unittest.TestCase): + """Base case: a temporary git repo standing in for the vault.""" + + def setUp(self) -> None: + self._tmp = tempfile.TemporaryDirectory() + self.root = Path(self._tmp.name).resolve() / "vault" + self.root.mkdir() + self.state = mcp.AppState( + root=self.root, graph_path=self.root / "index" / "graph.json" + ) + _git(self.root, "init", "-q", "-b", "main") + _git(self.root, "config", "user.email", "test@example.com") + _git(self.root, "config", "user.name", "Test") + _git(self.root, "config", "commit.gpgsign", "false") + (self.root / "README.md").write_text("# vault\n", encoding="utf-8") + _git(self.root, "add", "README.md") + _git(self.root, "commit", "-qm", "init") + + def tearDown(self) -> None: + self._tmp.cleanup() + + def call(self, name: str, **args) -> dict: + return mcp._call_tool(self.state, name, args) + + def structured(self, name: str, **args) -> dict: + return self.call(name, **args)["structuredContent"] + + +class TestToolDefinitions(unittest.TestCase): + def test_write_and_git_tools_are_advertised(self) -> None: + names = {tool["name"] for tool in mcp._tool_definitions()} + self.assertLessEqual( + {"write_note", "edit_note", "capture_raw", "git_commit", "git_push"}, + names, + ) + + def test_every_tool_has_a_schema_and_description(self) -> None: + for tool in mcp._tool_definitions(): + self.assertTrue(tool.get("description"), tool["name"]) + self.assertEqual(tool["inputSchema"]["type"], "object", tool["name"]) + + +class TestWriteNote(ToolTestCase): + def test_creates_note_and_parent_directories(self) -> None: + payload = self.structured( + "write_note", + path="03_resources/demo/index.md", + content="---\nid: resource/demo\n---\n\n# Demo\n", + ) + target = self.root / "03_resources/demo/index.md" + self.assertTrue(target.exists()) + self.assertIn("# Demo", target.read_text(encoding="utf-8")) + self.assertEqual(payload["path"], "03_resources/demo/index.md") + self.assertTrue(payload["created"]) + + def test_refuses_to_overwrite_existing_note_by_default(self) -> None: + self.call("write_note", path="note.md", content="first") + with self.assertRaises(FileExistsError): + self.call("write_note", path="note.md", content="second") + self.assertEqual((self.root / "note.md").read_text(encoding="utf-8"), "first") + + def test_overwrites_when_asked(self) -> None: + self.call("write_note", path="note.md", content="first") + payload = self.structured( + "write_note", path="note.md", content="second", overwrite=True + ) + self.assertFalse(payload["created"]) + self.assertEqual((self.root / "note.md").read_text(encoding="utf-8"), "second") + + def test_rejects_path_outside_the_vault(self) -> None: + with self.assertRaises(ValueError): + self.call("write_note", path="../escape.md", content="nope") + self.assertFalse((self.root.parent / "escape.md").exists()) + + def test_rejects_absolute_path(self) -> None: + with self.assertRaises(ValueError): + self.call("write_note", path="/tmp/escape.md", content="nope") + + def test_requires_path_and_content(self) -> None: + with self.assertRaises(ValueError): + self.call("write_note", content="orphan") + with self.assertRaises(ValueError): + self.call("write_note", path="note.md") + + def test_rejects_writes_into_the_generated_graph(self) -> None: + with self.assertRaises(ValueError): + self.call("write_note", path="index/graph.json", content="{}") + + +class TestEditNote(ToolTestCase): + def setUp(self) -> None: + super().setUp() + (self.root / "note.md").write_text("alpha beta alpha\n", encoding="utf-8") + + def test_replaces_a_unique_occurrence(self) -> None: + payload = self.structured( + "edit_note", path="note.md", old_string="beta", new_string="gamma" + ) + self.assertEqual(payload["replacements"], 1) + self.assertEqual( + (self.root / "note.md").read_text(encoding="utf-8"), "alpha gamma alpha\n" + ) + + def test_rejects_ambiguous_occurrence(self) -> None: + with self.assertRaises(ValueError): + self.call("edit_note", path="note.md", old_string="alpha", new_string="x") + self.assertEqual( + (self.root / "note.md").read_text(encoding="utf-8"), "alpha beta alpha\n" + ) + + def test_replaces_all_when_asked(self) -> None: + payload = self.structured( + "edit_note", + path="note.md", + old_string="alpha", + new_string="x", + replace_all=True, + ) + self.assertEqual(payload["replacements"], 2) + self.assertEqual( + (self.root / "note.md").read_text(encoding="utf-8"), "x beta x\n" + ) + + def test_errors_when_old_string_absent(self) -> None: + with self.assertRaises(ValueError): + self.call("edit_note", path="note.md", old_string="zeta", new_string="x") + + def test_errors_when_note_missing(self) -> None: + with self.assertRaises(FileNotFoundError): + self.call("edit_note", path="ghost.md", old_string="a", new_string="b") + + def test_rejects_identical_strings(self) -> None: + with self.assertRaises(ValueError): + self.call("edit_note", path="note.md", old_string="beta", new_string="beta") + + def test_rejects_path_outside_the_vault(self) -> None: + with self.assertRaises(ValueError): + self.call("edit_note", path="../../x.md", old_string="a", new_string="b") + + +class TestGitCommit(ToolTestCase): + def _log(self) -> str: + return _git(self.root, "log", "--format=%s", "-n", "1").stdout.strip() + + def test_stages_named_paths_and_commits(self) -> None: + self.call("write_note", path="03_resources/demo/index.md", content="# Demo\n") + self.call("write_note", path="03_resources/demo/topic.md", content="# Topic\n") + payload = self.structured( + "git_commit", + message="feat: ingest demo resource", + paths=["03_resources/demo/index.md", "03_resources/demo/topic.md"], + ) + self.assertEqual(payload["exit_code"], 0) + self.assertEqual(self._log(), "feat: ingest demo resource") + self.assertEqual(payload["committed_paths"], sorted(payload["committed_paths"])) + self.assertEqual( + _git(self.root, "status", "--porcelain").stdout.strip(), "" + ) + + def test_commits_only_the_named_paths(self) -> None: + self.call("write_note", path="wanted.md", content="a\n") + self.call("write_note", path="unwanted.md", content="b\n") + self.call("git_commit", message="feat: only wanted", paths=["wanted.md"]) + status = _git(self.root, "status", "--porcelain").stdout.split() + self.assertIn("unwanted.md", status) + self.assertNotIn("wanted.md", status) + + def test_requires_at_least_one_path(self) -> None: + with self.assertRaises(ValueError): + self.call("git_commit", message="feat: nothing", paths=[]) + + def test_rejects_wildcard_staging(self) -> None: + for bad in [".", "-A", "*", "./"]: + with self.assertRaises(ValueError): + self.call("git_commit", message="feat: broad", paths=[bad]) + + def test_requires_a_message(self) -> None: + self.call("write_note", path="note.md", content="a\n") + with self.assertRaises(ValueError): + self.call("git_commit", message="", paths=["note.md"]) + + def test_rejects_path_outside_the_vault(self) -> None: + with self.assertRaises(ValueError): + self.call("git_commit", message="feat: escape", paths=["../outside.md"]) + + def test_errors_when_path_does_not_exist(self) -> None: + with self.assertRaises(FileNotFoundError): + self.call("git_commit", message="feat: ghost", paths=["ghost.md"]) + + def test_reports_error_when_nothing_to_commit(self) -> None: + result = self.call("git_commit", message="feat: noop", paths=["README.md"]) + self.assertTrue(result.get("isError")) + self.assertNotEqual(result["structuredContent"]["exit_code"], 0) + + +class TestGitPush(ToolTestCase): + def setUp(self) -> None: + super().setUp() + self.remote = self.root.parent / "remote.git" + _git(self.root.parent, "init", "-q", "--bare", str(self.remote)) + _git(self.root, "remote", "add", "origin", str(self.remote)) + + def _remote_log(self) -> str: + return subprocess.run( + ["git", "log", "--format=%s", "-n", "1", "main"], + cwd=self.remote, + capture_output=True, + text=True, + ).stdout.strip() + + def test_pushes_current_branch(self) -> None: + self.call("write_note", path="note.md", content="a\n") + self.call("git_commit", message="feat: add note", paths=["note.md"]) + payload = self.structured("git_push", set_upstream=True) + self.assertEqual(payload["exit_code"], 0) + self.assertEqual(payload["branch"], "main") + self.assertEqual(self._remote_log(), "feat: add note") + + def test_reports_error_for_unknown_remote(self) -> None: + result = self.call("git_push", remote="nope") + self.assertTrue(result.get("isError")) + self.assertNotEqual(result["structuredContent"]["exit_code"], 0) + + def test_rejects_option_like_remote_or_branch(self) -> None: + with self.assertRaises(ValueError): + self.call("git_push", remote="--force") + with self.assertRaises(ValueError): + self.call("git_push", branch="--delete") + + +class _QuietHandler(http.server.SimpleHTTPRequestHandler): + """Serves a fixture directory without logging to stderr.""" + + def log_message(self, *args) -> None: # noqa: D102 - silence the test server + pass + + +class TestCaptureRaw(ToolTestCase): + """capture_raw preserves source bytes verbatim, local or downloaded.""" + + PDF_BYTES = b"%PDF-1.7\n\x00\x01\x02binary\xff\xfe payload\n%%EOF\n" + + def setUp(self) -> None: + super().setUp() + self.fixtures = Path(self._tmp.name) / "fixtures" + self.fixtures.mkdir() + self.pdf = self.fixtures / "paper.pdf" + self.pdf.write_bytes(self.PDF_BYTES) + + self.server = http.server.ThreadingHTTPServer( + ("127.0.0.1", 0), + lambda *a, **kw: _QuietHandler(*a, directory=str(self.fixtures), **kw), + ) + self.base_url = f"http://127.0.0.1:{self.server.server_address[1]}" + self.thread = threading.Thread(target=self.server.serve_forever, daemon=True) + self.thread.start() + + def tearDown(self) -> None: + self.server.shutdown() + self.server.server_close() + self.thread.join(timeout=5) + super().tearDown() + + def test_copies_a_local_binary_verbatim(self) -> None: + payload = self.structured( + "capture_raw", + path="03_resources/demo/raw/paper.pdf", + source_path=str(self.pdf), + ) + target = self.root / "03_resources/demo/raw/paper.pdf" + self.assertEqual(target.read_bytes(), self.PDF_BYTES) + self.assertEqual(payload["bytes"], len(self.PDF_BYTES)) + self.assertEqual( + payload["sha256"], hashlib.sha256(self.PDF_BYTES).hexdigest() + ) + self.assertEqual(payload["source"], str(self.pdf)) + self.assertTrue(payload["created"]) + + def test_downloads_a_binary_verbatim(self) -> None: + payload = self.structured( + "capture_raw", + path="03_resources/demo/raw/paper.pdf", + url=f"{self.base_url}/paper.pdf", + ) + target = self.root / "03_resources/demo/raw/paper.pdf" + self.assertEqual(target.read_bytes(), self.PDF_BYTES) + self.assertEqual( + payload["sha256"], hashlib.sha256(self.PDF_BYTES).hexdigest() + ) + + def test_refuses_to_overwrite_by_default(self) -> None: + self.call("capture_raw", path="raw/paper.pdf", source_path=str(self.pdf)) + other = self.fixtures / "other.pdf" + other.write_bytes(b"different") + with self.assertRaises(FileExistsError): + self.call("capture_raw", path="raw/paper.pdf", source_path=str(other)) + self.assertEqual( + (self.root / "raw/paper.pdf").read_bytes(), self.PDF_BYTES + ) + + def test_overwrites_when_asked(self) -> None: + self.call("capture_raw", path="raw/paper.pdf", source_path=str(self.pdf)) + other = self.fixtures / "other.pdf" + other.write_bytes(b"different") + payload = self.structured( + "capture_raw", + path="raw/paper.pdf", + source_path=str(other), + overwrite=True, + ) + self.assertFalse(payload["created"]) + self.assertEqual((self.root / "raw/paper.pdf").read_bytes(), b"different") + + def test_requires_exactly_one_source(self) -> None: + with self.assertRaises(ValueError): + self.call("capture_raw", path="raw/x.pdf") + with self.assertRaises(ValueError): + self.call( + "capture_raw", + path="raw/x.pdf", + url=f"{self.base_url}/paper.pdf", + source_path=str(self.pdf), + ) + self.assertFalse((self.root / "raw/x.pdf").exists()) + + def test_rejects_destination_outside_the_vault(self) -> None: + with self.assertRaises(ValueError): + self.call( + "capture_raw", path="../escape.pdf", source_path=str(self.pdf) + ) + self.assertFalse((self.root.parent / "escape.pdf").exists()) + + def test_rejects_non_http_url_scheme(self) -> None: + for bad in [f"file://{self.pdf}", "ftp://example.com/x.pdf", "data:,x"]: + with self.assertRaises(ValueError): + self.call("capture_raw", path="raw/x.pdf", url=bad) + + def test_errors_when_local_source_missing(self) -> None: + with self.assertRaises(FileNotFoundError): + self.call( + "capture_raw", + path="raw/x.pdf", + source_path=str(self.fixtures / "ghost.pdf"), + ) + + def test_errors_when_local_source_is_a_directory(self) -> None: + with self.assertRaises(ValueError): + self.call("capture_raw", path="raw/x.pdf", source_path=str(self.fixtures)) + + def test_download_failure_leaves_no_partial_file(self) -> None: + result = self.call( + "capture_raw", path="raw/x.pdf", url=f"{self.base_url}/missing.pdf" + ) + self.assertTrue(result.get("isError")) + self.assertFalse((self.root / "raw/x.pdf").exists()) + + def test_enforces_the_size_limit(self) -> None: + with self.assertRaises(ValueError): + self.call( + "capture_raw", + path="raw/x.pdf", + source_path=str(self.pdf), + max_bytes=4, + ) + self.assertFalse((self.root / "raw/x.pdf").exists()) + + def test_captured_binary_is_committable(self) -> None: + self.call( + "capture_raw", + path="03_resources/demo/raw/paper.pdf", + source_path=str(self.pdf), + ) + payload = self.structured( + "git_commit", + message="feat: ingest demo pdf", + paths=["03_resources/demo/raw/paper.pdf"], + ) + self.assertEqual(payload["exit_code"], 0) + + +class TestReadNoteBinary(ToolTestCase): + """read_note must say what to do instead of choking on binary bytes.""" + + def test_reports_binary_notes_instead_of_failing_to_decode(self) -> None: + target = self.root / "03_resources/demo/raw/paper.pdf" + target.parent.mkdir(parents=True) + target.write_bytes(b"%PDF-1.7\n\x00\x01\xff binary") + result = self.call("read_note", path="03_resources/demo/raw/paper.pdf") + payload = result["structuredContent"] + self.assertTrue(payload["binary"]) + self.assertNotIn("content", payload) + self.assertIn("sha256", payload) + self.assertIn(str(self.root / "03_resources/demo/raw/paper.pdf"), payload["hint"]) + + +if __name__ == "__main__": + unittest.main()