optimize agent-docs
This commit is contained in:
Binary file not shown.
473
skills/agent-docs/scripts/validate_agent_docs.py
Normal file
473
skills/agent-docs/scripts/validate_agent_docs.py
Normal file
@@ -0,0 +1,473 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate agent-doc structure and agent-doc skill-suite consistency."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from collections import deque
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
BACKTICK_MD_RE = re.compile(r"`([^`\n]+?\.md)`")
|
||||
H2_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE)
|
||||
FRONTMATTER_NAME_RE = re.compile(r"^name:\s*([A-Za-z0-9-]+)\s*$", re.MULTILINE)
|
||||
DEFAULT_PROMPT_RE = re.compile(r'^\s*default_prompt:\s*"([^"]+)"\s*$', re.MULTILINE)
|
||||
BULLET_RE = re.compile(r"^\s*-\s+(.+?)\s*$", re.MULTILINE)
|
||||
SECTION_RE = re.compile(r"^##\s+(.+?)\s*$", re.MULTILINE)
|
||||
STRONG_RULE_RE = re.compile(r"\b(read and follow|must|required|do not|always)\b", re.IGNORECASE)
|
||||
|
||||
RULE_REQUIRED_HEADINGS = {"Applies To", "Authority", "Checklist"}
|
||||
RULE_OPTIONAL_HEADINGS = {"Does Not Apply To", "Edge Cases", "Failure Modes"}
|
||||
INVENTORY_REQUIRED_HEADINGS = {
|
||||
"Applies To",
|
||||
"Maintenance Rules",
|
||||
"Reclassification Rules",
|
||||
"Classification Authority",
|
||||
"Entries",
|
||||
"Maintenance Checklist",
|
||||
}
|
||||
|
||||
|
||||
@dataclass
|
||||
class ValidationReport:
|
||||
errors: list[str] = field(default_factory=list)
|
||||
structural_warnings: list[str] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
notes: list[str] = field(default_factory=list)
|
||||
|
||||
def error(self, message: str) -> None:
|
||||
self.errors.append(message)
|
||||
|
||||
def warn(self, message: str) -> None:
|
||||
self.warnings.append(message)
|
||||
|
||||
def structural_warn(self, message: str) -> None:
|
||||
self.structural_warnings.append(message)
|
||||
|
||||
def note(self, message: str) -> None:
|
||||
self.notes.append(message)
|
||||
|
||||
|
||||
def read_text(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def normalize_heading(heading: str) -> str:
|
||||
return " ".join(heading.strip().split())
|
||||
|
||||
|
||||
def normalize_phrase(value: str) -> str:
|
||||
return " ".join(value.lower().split())
|
||||
|
||||
|
||||
def extract_h2_headings(text: str) -> set[str]:
|
||||
return {normalize_heading(match.group(1)) for match in H2_RE.finditer(text)}
|
||||
|
||||
|
||||
def extract_section_text(text: str, heading: str) -> str:
|
||||
matches = list(SECTION_RE.finditer(text))
|
||||
for index, match in enumerate(matches):
|
||||
if normalize_heading(match.group(1)) != heading:
|
||||
continue
|
||||
start = match.end()
|
||||
end = matches[index + 1].start() if index + 1 < len(matches) else len(text)
|
||||
return text[start:end].strip()
|
||||
return ""
|
||||
|
||||
|
||||
def extract_section_bullets(text: str, heading: str) -> list[str]:
|
||||
section = extract_section_text(text, heading)
|
||||
return [match.group(1).strip() for match in BULLET_RE.finditer(section)]
|
||||
|
||||
|
||||
def has_anchor_token(value: str) -> bool:
|
||||
return "`" in value or "/" in value or "." in value or ":" in value
|
||||
|
||||
|
||||
def is_vague_phrase(value: str) -> bool:
|
||||
normalized = normalize_phrase(value)
|
||||
vague_phrases = {
|
||||
"related tasks",
|
||||
"relevant tasks",
|
||||
"appropriate tasks",
|
||||
"matching tasks",
|
||||
"relevant work",
|
||||
"related work",
|
||||
"relevant code",
|
||||
"related code",
|
||||
"the code",
|
||||
"the docs",
|
||||
"documentation",
|
||||
"repository",
|
||||
"repo",
|
||||
"maintainers",
|
||||
"owners",
|
||||
"owner",
|
||||
}
|
||||
return normalized in vague_phrases
|
||||
|
||||
|
||||
def extract_strong_rule_lines(text: str) -> set[str]:
|
||||
strong_lines: set[str] = set()
|
||||
for raw_line in text.splitlines():
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or line.startswith("```"):
|
||||
continue
|
||||
if STRONG_RULE_RE.search(line):
|
||||
strong_lines.add(normalize_phrase(line))
|
||||
return strong_lines
|
||||
|
||||
|
||||
def extract_frontmatter_name(text: str) -> str | None:
|
||||
if not text.startswith("---\n"):
|
||||
return None
|
||||
end = text.find("\n---\n", 4)
|
||||
if end == -1:
|
||||
return None
|
||||
frontmatter = text[4:end]
|
||||
match = FRONTMATTER_NAME_RE.search(frontmatter)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def extract_default_prompt(text: str) -> str | None:
|
||||
match = DEFAULT_PROMPT_RE.search(text)
|
||||
return match.group(1) if match else None
|
||||
|
||||
|
||||
def md_reference_targets(text: str) -> list[Path]:
|
||||
targets: list[Path] = []
|
||||
for raw in BACKTICK_MD_RE.findall(text):
|
||||
candidate = Path(raw.strip())
|
||||
if candidate.name == "AGENTS.md" or "agent-docs" in candidate.parts:
|
||||
targets.append(candidate)
|
||||
return targets
|
||||
|
||||
|
||||
def agent_doc_files(repo_root: Path) -> list[Path]:
|
||||
docs = []
|
||||
for path in repo_root.rglob("*.md"):
|
||||
relative = path.relative_to(repo_root)
|
||||
if relative.name == "AGENTS.md" and relative.parent == Path("."):
|
||||
continue
|
||||
if "agent-docs" in relative.parts:
|
||||
docs.append(path)
|
||||
return sorted(docs)
|
||||
|
||||
|
||||
def is_inventory_doc(headings: set[str]) -> bool:
|
||||
return "Entries" in headings or "Maintenance Rules" in headings
|
||||
|
||||
|
||||
def validate_doc_shape(path: Path, report: ValidationReport, display_path: Path | None = None) -> None:
|
||||
text = read_text(path)
|
||||
headings = extract_h2_headings(text)
|
||||
label = str(display_path or path)
|
||||
|
||||
if is_inventory_doc(headings):
|
||||
missing = sorted(INVENTORY_REQUIRED_HEADINGS - headings)
|
||||
if missing:
|
||||
report.error(
|
||||
f"{label}: inventory doc is missing required headings: {', '.join(missing)}"
|
||||
)
|
||||
else:
|
||||
missing = sorted(RULE_REQUIRED_HEADINGS - headings)
|
||||
if missing:
|
||||
report.error(
|
||||
f"{label}: rule doc is missing required headings: {', '.join(missing)}"
|
||||
)
|
||||
rule_sections = headings - RULE_REQUIRED_HEADINGS - RULE_OPTIONAL_HEADINGS
|
||||
if not rule_sections:
|
||||
report.error(
|
||||
f"{label}: rule doc needs at least one topic-specific rule section beyond scope, authority, and checklist headings"
|
||||
)
|
||||
|
||||
applies_to = extract_section_bullets(text, "Applies To")
|
||||
if not applies_to:
|
||||
report.warn(f"{label}: `Applies To` has no bullet entries")
|
||||
for bullet in applies_to:
|
||||
if is_vague_phrase(bullet):
|
||||
report.warn(f"{label}: vague `Applies To` bullet `{bullet}` should be more specific")
|
||||
|
||||
for bullet in extract_section_bullets(text, "Authority"):
|
||||
if is_vague_phrase(bullet) or (len(bullet) < 18 and not has_anchor_token(bullet)):
|
||||
report.warn(
|
||||
f"{label}: vague `Authority` bullet `{bullet}` should name a stronger source-of-truth anchor"
|
||||
)
|
||||
|
||||
|
||||
def validate_repo_graph(repo_root: Path, report: ValidationReport, max_depth: int, max_out_degree: int) -> None:
|
||||
root_agents = repo_root / "AGENTS.md"
|
||||
if not root_agents.exists():
|
||||
report.error(f"{root_agents}: missing root AGENTS.md")
|
||||
return
|
||||
|
||||
docs = agent_doc_files(repo_root)
|
||||
nodes = [root_agents, *docs]
|
||||
node_set = set(nodes)
|
||||
edges: dict[Path, set[Path]] = {node: set() for node in nodes}
|
||||
|
||||
for node in nodes:
|
||||
text = read_text(node)
|
||||
for target in md_reference_targets(text):
|
||||
resolved = (repo_root / target).resolve()
|
||||
try:
|
||||
resolved.relative_to(repo_root.resolve())
|
||||
except ValueError:
|
||||
report.error(
|
||||
f"{node.relative_to(repo_root)}: referenced path escapes repo root: {target}"
|
||||
)
|
||||
continue
|
||||
if not resolved.exists():
|
||||
report.error(
|
||||
f"{node.relative_to(repo_root)}: stale reference to missing file `{target.as_posix()}`"
|
||||
)
|
||||
continue
|
||||
if resolved in node_set:
|
||||
edges[node].add(resolved)
|
||||
|
||||
queue: deque[Path] = deque([root_agents])
|
||||
depths: dict[Path, int] = {root_agents: 0}
|
||||
while queue:
|
||||
current = queue.popleft()
|
||||
for child in sorted(edges[current]):
|
||||
if child not in depths:
|
||||
depths[child] = depths[current] + 1
|
||||
queue.append(child)
|
||||
|
||||
for doc in docs:
|
||||
if doc not in depths:
|
||||
report.error(
|
||||
f"{doc.relative_to(repo_root)}: active child doc is not reachable from root AGENTS.md"
|
||||
)
|
||||
|
||||
max_observed_depth = max(depths.values(), default=0)
|
||||
report.note(f"repo graph depth: {max_observed_depth}")
|
||||
if max_observed_depth > max_depth:
|
||||
report.structural_warn(
|
||||
f"repo graph depth {max_observed_depth} exceeds recommended maximum {max_depth}"
|
||||
)
|
||||
|
||||
for node, children in sorted(edges.items()):
|
||||
if len(children) > max_out_degree:
|
||||
report.structural_warn(
|
||||
f"{node.relative_to(repo_root)}: out-degree {len(children)} exceeds recommended maximum {max_out_degree}"
|
||||
)
|
||||
|
||||
visiting: set[Path] = set()
|
||||
visited: set[Path] = set()
|
||||
stack: list[Path] = []
|
||||
|
||||
def dfs(node: Path) -> None:
|
||||
visiting.add(node)
|
||||
stack.append(node)
|
||||
for child in sorted(edges[node]):
|
||||
if child in visiting:
|
||||
start = stack.index(child)
|
||||
cycle = stack[start:] + [child]
|
||||
cycle_text = " -> ".join(str(item.relative_to(repo_root)) for item in cycle)
|
||||
report.error(f"cycle detected: {cycle_text}")
|
||||
continue
|
||||
if child not in visited:
|
||||
dfs(child)
|
||||
stack.pop()
|
||||
visiting.remove(node)
|
||||
visited.add(node)
|
||||
|
||||
dfs(root_agents)
|
||||
|
||||
applies_to_index: dict[str, list[Path]] = {}
|
||||
doc_profiles: dict[Path, dict[str, set[str]]] = {}
|
||||
for doc in docs:
|
||||
validate_doc_shape(doc, report, doc.relative_to(repo_root))
|
||||
doc_text = read_text(doc)
|
||||
applies_to_bullets = {
|
||||
normalize_phrase(bullet) for bullet in extract_section_bullets(doc_text, "Applies To")
|
||||
}
|
||||
authority_bullets = {
|
||||
normalize_phrase(bullet) for bullet in extract_section_bullets(doc_text, "Authority")
|
||||
}
|
||||
doc_profiles[doc] = {
|
||||
"applies_to": applies_to_bullets,
|
||||
"authority": authority_bullets,
|
||||
"strong_rules": extract_strong_rule_lines(doc_text),
|
||||
}
|
||||
|
||||
for bullet in applies_to_bullets:
|
||||
normalized = bullet
|
||||
applies_to_index.setdefault(normalized, []).append(doc.relative_to(repo_root))
|
||||
|
||||
nonempty_lines = [line for line in doc_text.splitlines() if line.strip()]
|
||||
reference_lines = [line for line in nonempty_lines if ".md`" in line or ".md" in line]
|
||||
if len(edges[doc]) >= 3 and nonempty_lines:
|
||||
reference_ratio = len(reference_lines) / len(nonempty_lines)
|
||||
if reference_ratio > 0.35 and len(extract_h2_headings(doc_text)) <= 4:
|
||||
report.structural_warn(
|
||||
f"{doc.relative_to(repo_root)}: high reference density suggests secondary-router behavior"
|
||||
)
|
||||
|
||||
for applies_to, owners in sorted(applies_to_index.items()):
|
||||
unique_owners = sorted({owner.as_posix() for owner in owners})
|
||||
if len(unique_owners) > 1 and not is_vague_phrase(applies_to):
|
||||
report.warn(
|
||||
f"shared `Applies To` bullet `{applies_to}` appears in multiple docs: {', '.join(unique_owners)}"
|
||||
)
|
||||
|
||||
sorted_docs = sorted(doc_profiles)
|
||||
for index, left_doc in enumerate(sorted_docs):
|
||||
left_profile = doc_profiles[left_doc]
|
||||
for right_doc in sorted_docs[index + 1 :]:
|
||||
right_profile = doc_profiles[right_doc]
|
||||
shared_applies = {
|
||||
value
|
||||
for value in left_profile["applies_to"] & right_profile["applies_to"]
|
||||
if not is_vague_phrase(value)
|
||||
}
|
||||
if not shared_applies:
|
||||
continue
|
||||
|
||||
shared_authority = {
|
||||
value
|
||||
for value in left_profile["authority"] & right_profile["authority"]
|
||||
if value and not is_vague_phrase(value)
|
||||
}
|
||||
shared_rules = left_profile["strong_rules"] & right_profile["strong_rules"]
|
||||
|
||||
if shared_authority:
|
||||
report.warn(
|
||||
"possible duplicate authority between "
|
||||
f"{left_doc.relative_to(repo_root)} and {right_doc.relative_to(repo_root)}: "
|
||||
f"shared scope {', '.join(sorted(shared_applies))}; shared authority {', '.join(sorted(shared_authority))}"
|
||||
)
|
||||
continue
|
||||
|
||||
if shared_rules:
|
||||
report.warn(
|
||||
"possible overlapping rule ownership between "
|
||||
f"{left_doc.relative_to(repo_root)} and {right_doc.relative_to(repo_root)}: "
|
||||
f"shared scope {', '.join(sorted(shared_applies))}; shared strong rule signals detected"
|
||||
)
|
||||
|
||||
|
||||
def validate_skill_suite(skill_suite_root: Path, report: ValidationReport) -> None:
|
||||
if not skill_suite_root.exists():
|
||||
report.error(f"{skill_suite_root}: missing skill-suite root")
|
||||
return
|
||||
|
||||
skill_dirs = sorted(
|
||||
path
|
||||
for path in skill_suite_root.iterdir()
|
||||
if path.is_dir() and path.name.startswith("agent-docs") and (path / "SKILL.md").exists()
|
||||
)
|
||||
if not skill_dirs:
|
||||
report.error(f"{skill_suite_root}: no agent-docs skill directories found")
|
||||
return
|
||||
|
||||
skill_texts: dict[str, str] = {}
|
||||
for skill_dir in skill_dirs:
|
||||
skill_md = skill_dir / "SKILL.md"
|
||||
text = read_text(skill_md)
|
||||
skill_texts[skill_dir.name] = text
|
||||
skill_name = extract_frontmatter_name(text)
|
||||
if not skill_name:
|
||||
report.error(f"{skill_md}: missing frontmatter name")
|
||||
continue
|
||||
if skill_name != skill_dir.name:
|
||||
report.error(f"{skill_md}: frontmatter name `{skill_name}` does not match directory `{skill_dir.name}`")
|
||||
|
||||
prompt_file = skill_dir / "agents" / "openai.yaml"
|
||||
if prompt_file.exists():
|
||||
prompt_text = read_text(prompt_file)
|
||||
default_prompt = extract_default_prompt(prompt_text)
|
||||
if not default_prompt:
|
||||
report.error(f"{prompt_file}: missing default_prompt")
|
||||
elif f"${skill_name}" not in default_prompt:
|
||||
report.error(
|
||||
f"{prompt_file}: default_prompt does not mention `${skill_name}`"
|
||||
)
|
||||
|
||||
if "bluecraft-agentic-docs" in text:
|
||||
report.error(f"{skill_md}: stale reference to `bluecraft-agentic-docs`")
|
||||
|
||||
refs_dir = skill_suite_root / "agent-docs" / "references"
|
||||
if refs_dir.exists():
|
||||
searchable_texts = [*skill_texts.values()]
|
||||
for skill_dir in skill_dirs:
|
||||
prompt_file = skill_dir / "agents" / "openai.yaml"
|
||||
if prompt_file.exists():
|
||||
searchable_texts.append(read_text(prompt_file))
|
||||
for ref_file in sorted(path for path in refs_dir.iterdir() if path.is_file()):
|
||||
if not any(ref_file.name in text for text in searchable_texts):
|
||||
report.error(
|
||||
f"{ref_file}: no skill file or prompt directly references this shared reference"
|
||||
)
|
||||
|
||||
max_prompt_length = 220
|
||||
for skill_dir in skill_dirs:
|
||||
prompt_file = skill_dir / "agents" / "openai.yaml"
|
||||
if not prompt_file.exists():
|
||||
continue
|
||||
default_prompt = extract_default_prompt(read_text(prompt_file))
|
||||
if default_prompt and len(default_prompt) > max_prompt_length:
|
||||
report.warn(
|
||||
f"{prompt_file}: default_prompt length {len(default_prompt)} exceeds recommended maximum {max_prompt_length}"
|
||||
)
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Validate AGENTS.md / agent-docs structure and agent-docs skill-suite consistency."
|
||||
)
|
||||
parser.add_argument("--repo-root", type=Path, help="Repository root containing AGENTS.md and agent-docs/")
|
||||
parser.add_argument("--skill-suite-root", type=Path, help="Root containing the agent-docs skill directories")
|
||||
parser.add_argument("--max-depth", type=int, default=3, help="Recommended maximum root-to-doc routing depth")
|
||||
parser.add_argument(
|
||||
"--max-out-degree",
|
||||
type=int,
|
||||
default=7,
|
||||
help="Recommended maximum number of child-doc references from one doc",
|
||||
)
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
if not args.repo_root and not args.skill_suite_root:
|
||||
parser.error("at least one of --repo-root or --skill-suite-root is required")
|
||||
|
||||
report = ValidationReport()
|
||||
if args.repo_root:
|
||||
validate_repo_graph(args.repo_root.resolve(), report, args.max_depth, args.max_out_degree)
|
||||
if args.skill_suite_root:
|
||||
validate_skill_suite(args.skill_suite_root.resolve(), report)
|
||||
|
||||
for note in report.notes:
|
||||
print(f"NOTE: {note}")
|
||||
for structural_warning in report.structural_warnings:
|
||||
print(f"STRUCTURAL-WARNING: {structural_warning}")
|
||||
for warning in report.warnings:
|
||||
print(f"WARNING: {warning}")
|
||||
for error in report.errors:
|
||||
print(f"ERROR: {error}")
|
||||
|
||||
if report.errors:
|
||||
print(
|
||||
"Validation failed with "
|
||||
f"{len(report.errors)} error(s), "
|
||||
f"{len(report.structural_warnings)} structural warning(s), and "
|
||||
f"{len(report.warnings)} warning(s)."
|
||||
)
|
||||
return 1
|
||||
|
||||
print(
|
||||
"Validation passed with "
|
||||
f"{len(report.structural_warnings)} structural warning(s) and "
|
||||
f"{len(report.warnings)} warning(s)."
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user