feat(okf-hub): Fase F — converter fence-aware (path-style/acentos/pipe escapado) + validador fence-aware

2026-06-28 22:39:23 +01:00
parent 0094b45bcf
commit 3752238699
4 changed files with 233 additions and 107 deletions
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+"""layer raw|wiki nos .md do Dev (código: docs=wiki, efémeros/testes/samples=raw)."""
+import sys,re
+from pathlib import Path
+ROOT=Path("/media/ealmeida/Dados/Dev"); DRY="--dry-run" in sys.argv
+EXCL_DIR=re.compile(r'/(node_modules|\.git|venv|\.venv|vendor|dist|build|site-packages|__pycache__|target|3rdparty)/')
+RAW=re.compile(r'(worklog|/logs?/|sess[aã]o|di[aá]rio|checkup|deep-research|/fontes?/|pesquisa|/tests?/|/__tests__/|/fixtures?/|/examples?/|/samples?/|/test-data/|/\.cache/|CHANGELOG-old|reuni[aã]o)', re.I)
+add=0
+for f in ROOT.rglob("*.md"):
+    s=str(f)
+    if EXCL_DIR.search(s) or any(p.startswith(".") for p in f.parts): continue
+    if f.name in ("index.md","log.md"): continue
+    try: t=f.read_text(encoding="utf-8")
+    except: continue
+    if not t.startswith("---"): continue
+    e=t.find("\n---",3)
+    if e<0: continue
+    if re.search(r'^layer:',t[3:e],re.M): continue
+    layer = "raw" if RAW.search(s) else "wiki"
+    if not DRY: f.write_text(t[:e]+f"\nlayer: {layer}"+t[e:],encoding="utf-8")
+    add+=1
+print(f"{'[DRY] ' if DRY else ''}layer adicionado: {add}")
@@ -1,153 +1,241 @@
 #!/usr/bin/env python3
 """
-okf-convert-wikilinks.py — Fase 3: Converte [[wikilinks]] → [texto](path.md) nos index.md
-OKF §5: links bundle-relative para navegação entre conceitos
+okf-convert-wikilinks.py — Fase F: converte [[wikilinks]] → [texto](path.md) no CORPO
+de notas `layer: wiki`, para compatibilidade com agentes CLI (que não resolvem [[ ]]).

-Âmbito: apenas ficheiros index.md (navegação)
-Corpo de documentos (PROC, QR, etc.) mantém wikilinks — OKF tolera e Obsidian renderiza ambos.
+Propriedades de segurança:
+  - FENCE-AWARE: nunca toca em code fences (``` ~~~) nem inline `code` — preserva
+    sintaxe bash `[[ -f x ]]` e exemplos de documentação.
+  - Resolução robusta: mesmo-directório → path-style (relativo, depois vault-root,
+    depois basename único) → stem global único. Acentos normalizados (NFKD), pipe
+    escapado `\\|` tratado, anchors `#`/`^` preservados no URL.
+  - Conservador: alvos ambíguos (stem repetido) ou não encontrados ficam [[ ]] e são
+    reportados — NUNCA se adivinha um destino.
+  - Exclusões alinhadas com okf-validate.sh (MEMORY/CLAUDE/AGENTS, 90-Templates, etc).

-Uso:
-  python3 okf-convert-wikilinks.py [--dry-run] [--dir /path/to/Hub]
-
-Criado: 28-06-2026
+Uso: python3 okf-convert-wikilinks.py [--dry-run] [--dir=/path/Hub] [--include-index]
 """
-
 import os
 import re
 import sys
+import unicodedata
 from pathlib import Path
+from collections import defaultdict, Counter

 HUB_DEFAULT = "/media/ealmeida/Dados/Hub"

-EXCLUDE_DIRS = {".stversions", "node_modules", ".git", ".obsidian", ".trash"}
+EXCLUDE_DIRS = {
+    ".stversions", "node_modules", ".git", ".obsidian", ".trash", ".ijfw", "ijfw",
+    ".github", ".wayland", ".hermes", ".vscode", ".cursor", ".gstack",
+    "_templates", "99-Arquivo", "90-Templates",
+}
+EXCLUDE_FILES = {"MEMORY.md", "CLAUDE.md", "GEMINI.md", "AGENTS.md", "copilot-instructions.md"}

-# Padrão wikilink: [[NomeFicheiro]] ou [[NomeFicheiro|Alias]]
-WIKILINK_RE = re.compile(r'\[\[([^\]|]+)(?:\|([^\]]+))?\]\]')
+WIKILINK_RE = re.compile(r'\[\[([^\]]+?)\]\]')
+FENCE_RE = re.compile(r'^(\s*)(```|~~~)')
+FM_RE = re.compile(r'^---\n(.*?)\n---', re.DOTALL)
+SKIP_PROSE = {"", " ", "...", "wikilinks", "Wikilinks"}


-def build_file_index(hub: Path) -> dict:
-    """Constrói índice nome→path para resolução de wikilinks."""
-    index = {}  # stem → Path relativo ao hub
+def fold(s: str) -> str:
+    return "".join(c for c in unicodedata.normalize("NFKD", s) if not unicodedata.combining(c)).lower()
+
+
+def is_excluded(rel: str) -> bool:
+    base = Path(rel).name
+    if base in EXCLUDE_FILES:
+        return True
+    if re.match(r'MEMORY-.*\.md$', base):
+        return True
+    return False
+
+
+def layer_of(text: str):
+    m = FM_RE.match(text)
+    if not m:
+        return None
+    lm = re.search(r'^layer:\s*(\w+)', m.group(1), re.M)
+    return lm.group(1) if lm else None
+
+
+def build_stem_index(hub: Path) -> dict:
+    """folded stem → [rel Path, ...] (lista para detectar ambiguidade)."""
+    idx = defaultdict(list)
    for root, dirs, files in os.walk(hub):
        dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS and not d.startswith(".")]
-        for fname in files:
-            if fname.endswith(".md"):
-                fp = Path(root) / fname
-                stem = fp.stem.lower()
-                rel = fp.relative_to(hub)
-                # Guardar o primeiro match (mais provável no vault activo)
-                if stem not in index:
-                    index[stem] = rel
-                # Também indexar o nome completo sem extensão
-                full_name = fname.lower()
-                if full_name not in index:
-                    index[full_name] = rel
-    return index
+        for f in files:
+            if f.endswith(".md"):
+                fp = Path(root) / f
+                idx[fold(fp.stem)].append(fp.relative_to(hub))
+    return idx


-def resolve_wikilink(target: str, current_file: Path, file_index: dict, hub: Path) -> str:
-    """Resolve [[target]] para um caminho relativo ao ficheiro actual."""
-    # Limpar o target (remover ^anchor, #heading, etc.)
-    target_clean = re.split(r'[#^]', target)[0].strip()
-    target_lower = target_clean.lower()
-    target_with_ext = target_lower + ".md" if not target_lower.endswith(".md") else target_lower
-
-    # Tentar resolver
-    resolved = file_index.get(target_with_ext) or file_index.get(target_lower)
-
-    if resolved:
-        # Calcular path relativo a partir do directório do ficheiro actual
-        try:
-            rel_path = os.path.relpath(hub / resolved, current_file.parent)
-            return rel_path.replace("\\", "/")
-        except Exception:
-            return str(resolved)
+def ci_dir_lookup(dirpath: Path, basename: str, hub: Path):
+    """Procura case-insensitive por basename(.md) em dirpath. Devolve rel Path ou None."""
+    if not dirpath.exists():
+        return None
+    want = fold(basename if basename.endswith(".md") else basename + ".md")
+    try:
+        for e in dirpath.iterdir():
+            if e.is_file() and fold(e.name) == want:
+                return e.relative_to(hub)
+    except Exception:
+        pass
    return None


-def convert_wikilinks_in_file(filepath: Path, file_index: dict, hub: Path, dry_run: bool) -> dict:
-    """Converte wikilinks no ficheiro. Retorna estatísticas."""
-    result = {"file": str(filepath.relative_to(hub)), "converted": 0, "unresolved": [], "action": "skip"}
-
-    try:
-        content = filepath.read_text(encoding="utf-8")
-    except Exception as e:
-        result["action"] = "error"
-        result["error"] = str(e)
-        return result
-
-    if "[[" not in content:
-        result["action"] = "no_wikilinks"
-        return result
-
-    def replace_wikilink(m):
-        target = m.group(1)
-        alias = m.group(2)
-        display = alias if alias else target
-
-        resolved_path = resolve_wikilink(target, filepath, file_index, hub)
-        if resolved_path:
-            result["converted"] += 1
-            return f"[{display}]({resolved_path})"
-        else:
-            # Manter como wikilink se não resolvível
-            result["unresolved"].append(target)
-            return m.group(0)
-
-    new_content = WIKILINK_RE.sub(replace_wikilink, content)
-
-    if new_content != content:
-        result["action"] = "converted"
-        if not dry_run:
-            filepath.write_text(new_content, encoding="utf-8")
+def resolve(target: str, current: Path, hub: Path, stem_idx: dict):
+    """Devolve (relpath, anchor) ou None. 'AMBIG' como relpath sinaliza ambiguidade."""
+    raw = target.strip()
+    split = re.split(r'(?=[#^])', raw, maxsplit=1)
+    name = split[0].strip().rstrip("\\").strip()
+    anchor = raw[len(split[0]):].strip() if len(split) > 1 else ""
+    if not name:
+        return None
+    cand = None
+    if "/" in name:
+        bn = Path(name).name
+        td = (current.parent / Path(name).parent).resolve()
+        if td == hub or hub in td.parents:
+            cand = ci_dir_lookup(td, bn, hub)
+        if cand is None:
+            td2 = (hub / Path(name).parent).resolve()
+            if td2 == hub or hub in td2.parents:
+                cand = ci_dir_lookup(td2, bn, hub)
+        if cand is None:
+            lst = stem_idx.get(fold(Path(bn).stem), [])
+            if len(lst) == 1:
+                cand = lst[0]
    else:
-        result["action"] = "no_changes"
+        cand = ci_dir_lookup(current.parent, name, hub)
+        if cand is None:
+            lst = stem_idx.get(fold(Path(name).stem), [])
+            if len(lst) == 1:
+                cand = lst[0]
+            elif len(lst) > 1:
+                return ("AMBIG", "")
+    if cand is None:
+        return None
+    relp = os.path.relpath(hub / cand, current.parent).replace("\\", "/")
+    return (relp, anchor)

-    return result
+
+def convert_file(fp: Path, hub: Path, stem_idx: dict, stats: dict) -> str:
+    content = fp.read_text(encoding="utf-8", errors="replace")
+    lines = content.split("\n")
+    out = []
+    in_fence = False
+    fmark = None
+    fm_end = -1
+    if lines and lines[0].strip() == "---":
+        for i in range(1, len(lines)):
+            if lines[i].strip() == "---":
+                fm_end = i
+                break
+    for idx, line in enumerate(lines):
+        if idx <= fm_end:
+            out.append(line)
+            continue
+        fm = FENCE_RE.match(line)
+        if not in_fence and fm:
+            in_fence = True
+            fmark = fm.group(2)
+            out.append(line)
+            continue
+        if in_fence:
+            if line.strip().startswith(fmark):
+                in_fence = False
+            out.append(line)
+            continue
+        parts = re.split(r'(`[^`]*`)', line)
+        for j, seg in enumerate(parts):
+            if seg.startswith("`"):
+                continue
+
+            def repl(mm):
+                inner = mm.group(1)
+                if "|" in inner:
+                    tgt, alias = inner.split("|", 1)
+                    tgt = tgt.strip().rstrip("\\").strip()
+                    alias = alias.strip()
+                else:
+                    tgt, alias = inner.strip(), None
+                if tgt in SKIP_PROSE:
+                    stats["prose"] += 1
+                    return mm.group(0)
+                r = resolve(tgt, fp, hub, stem_idx)
+                if r is None:
+                    stats["unresolved"][tgt] += 1
+                    return mm.group(0)
+                if r[0] == "AMBIG":
+                    stats["ambiguous"][tgt] += 1
+                    return mm.group(0)
+                relp, anchor = r
+                disp = alias if alias else tgt
+                url = relp + ("#" + anchor.lstrip("#^") if anchor else "")
+                stats["converted"] += 1
+                return f"[{disp}]({url})"
+
+            parts[j] = WIKILINK_RE.sub(repl, seg)
+        out.append("".join(parts))
+    return "\n".join(out)


 def main():
    dry_run = "--dry-run" in sys.argv
+    include_index = "--include-index" in sys.argv
    hub = Path(HUB_DEFAULT)
    for arg in sys.argv[1:]:
        if arg.startswith("--dir="):
            hub = Path(arg[6:])
-
    if not hub.exists():
        print(f"ERRO: Hub não encontrado em {hub}", file=sys.stderr)
        sys.exit(1)

-    print(f"{'[DRY-RUN] ' if dry_run else ''}A construir índice de ficheiros…")
-    file_index = build_file_index(hub)
-    print(f"  {len(file_index)} ficheiros indexados")
+    print(f"{'[DRY-RUN] ' if dry_run else ''}A indexar ficheiros…")
+    stem_idx = build_stem_index(hub)
+    print(f"  {len(stem_idx)} stems indexados")

-    print(f"A converter wikilinks nos index.md…")
-    total_converted = 0
-    total_unresolved = []
+    stats = {"converted": 0, "prose": 0, "unresolved": Counter(), "ambiguous": Counter()}
    files_changed = 0
-
    for root, dirs, files in os.walk(hub):
        dirs[:] = [d for d in dirs if d not in EXCLUDE_DIRS and not d.startswith(".")]
-        for fname in files:
-            if fname != "index.md":
+        for f in files:
+            if not f.endswith(".md"):
                continue
-            filepath = Path(root) / fname
-            result = convert_wikilinks_in_file(filepath, file_index, hub, dry_run)
-
-            if result["action"] == "converted":
+            if f == "index.md" and not include_index:
+                continue
+            rel = str((Path(root) / f).relative_to(hub))
+            if is_excluded(rel):
+                continue
+            fp = Path(root) / f
+            try:
+                txt = fp.read_text(encoding="utf-8", errors="replace")
+            except Exception:
+                continue
+            if layer_of(txt) != "wiki":
+                continue
+            if "[[" not in txt:
+                continue
+            new = convert_file(fp, hub, stem_idx, stats)
+            if new != txt:
                files_changed += 1
-                total_converted += result["converted"]
-                total_unresolved.extend(result["unresolved"])
-                print(f"  [OK] {result['file']}: {result['converted']} convertidos"
-                      + (f", {len(result['unresolved'])} não resolvidos" if result["unresolved"] else ""))
-            elif result["action"] == "error":
-                print(f"  [ERRO] {result['file']}: {result.get('error')}")
+                if not dry_run:
+                    fp.write_text(new, encoding="utf-8")
+                print(f"  [{'DRY' if dry_run else 'OK'}] {rel}")

-    print(f"\n=== Resultado ===")
+    print("\n=== Resultado ===")
    print(f"Ficheiros alterados: {files_changed}")
-    print(f"Wikilinks convertidos: {total_converted}")
-    if total_unresolved:
-        print(f"Não resolvidos ({len(total_unresolved)}): {', '.join(set(total_unresolved))[:200]}")
+    print(f"Wikilinks convertidos: {stats['converted']}")
+    print(f"Prosa-sobre-sintaxe saltada: {stats['prose']}")
+    amb = sum(stats["ambiguous"].values())
+    unr = sum(stats["unresolved"].values())
+    if amb:
+        print(f"AMBÍGUOS (deixados): {amb} → {dict(stats['ambiguous'].most_common(10))}")
+    if unr:
+        print(f"NÃO-RESOLVÍVEIS (refs partidas, deixados como [[ ]]): {unr} (distintos {len(stats['unresolved'])})")
+        for t, n in stats["unresolved"].most_common(30):
+            print(f"   {n:3} [[{t}]]")


 if __name__ == "__main__":
@@ -27,6 +27,15 @@ EXCLUDE_DIRS = {
    ".obsidian",
    ".trash",
    "99-Arquivo",
+    # dependências / builds / caches (relevante p/ Dev) — dot-dirs já são saltados
+    "venv",
+    "vendor",
+    "dist",
+    "build",
+    "site-packages",
+    "target",
+    "__pycache__",
+    "3rdparty",
 }

 # Ficheiros reservados OKF — sem frontmatter obrigatório
@@ -213,7 +213,14 @@ validate_file() {
    # Check for wikilinks in body (wiki layer)
    if echo "$content" | grep -q '^layer: wiki'; then
        local body
-        body="$(sed -n '/^---$/,/^---$/d; p' "$f")"
+        body="$(awk '
+            NR==1 && $0=="---" {infm=1; next}
+            infm && $0=="---" {infm=0; next}
+            infm {next}
+            /^[[:space:]]*```/ || /^[[:space:]]*~~~/ {infence=!infence; next}
+            infence {next}
+            {gsub(/`[^`]*`/,""); print}
+        ' "$f")"
        if echo "$body" | grep -q '\[\['; then
            echo -e "${YEL}WARN${NC}  $rel [wiki]: contains wikilinks [[ ]] — convert to [text](path)"
            ((WARNINGS++)) || true