From fac643fdc6214b1f0a060e01cb03c7356ae00fa4 Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sun, 16 Aug 2026 17:17:55 +0700 Subject: [PATCH 1/2] Reference children docs by path in overview prompts instead of inlining MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Parent/repo overview prompts embedded the entire module tree — every module's components list — as indent-4 JSON plus the full markdown of every child doc. On large repos (wazuh: ~2.2M chars) that exceeds codex's hard per-turn input cap of 1,048,576 chars, so every overview call failed with input_too_large, surfaced misleadingly as "LLM returned empty content". - build_overview_structure now strips components lists recursively and sets an absolute docs_path per child instead of inlining docs (None when the doc is missing). Wazuh-scale prompts drop from ~2.2M to ~83K chars. - REPO_OVERVIEW_PROMPT / MODULE_OVERVIEW_PROMPT instruct the agent to read the referenced files itself (backend.complete runs with ToolGroup.READER, so file reads are available). Placeholders are unchanged, keeping the MCP prompt_server rendering path working. - Pin coding-agent-wrapper to the fork branch carrying the matching codex provider fixes (stdin prompts, resume sandbox flag, and surfacing CLI failures that were silently returned as empty turns). Co-Authored-By: Claude Fable 5 --- codewiki/src/be/documentation_generator.py | 37 +++++++--- codewiki/src/be/prompt_template.py | 8 ++- pyproject.toml | 2 +- tests/test_overview_structure.py | 78 ++++++++++++++++++++++ 4 files changed, 114 insertions(+), 11 deletions(-) create mode 100644 tests/test_overview_structure.py diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index a91ee426..617292f1 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -117,9 +117,17 @@ def is_leaf_module(self, module_info: Dict[str, Any]) -> bool: def build_overview_structure(self, module_tree: Dict[str, Any], module_path: List[str], working_dir: str) -> Dict[str, Any]: - """Build structure for overview generation with 1-depth children docs and target indicator.""" - + """Build structure for overview generation with 1-depth children doc paths and target indicator. + + Children docs are referenced by absolute file path (``docs_path``) + rather than inlined, and ``components`` lists are stripped: inlining + the full tree plus docs blew past provider input caps (codex rejects + turns over 1,048,576 chars) on large repos. The overview agent reads + the referenced files itself. + """ + processed_module_tree = deepcopy(module_tree) + self._strip_components(processed_module_tree) module_info = processed_module_tree for path_part in module_path: module_info = module_info[path_part] @@ -134,13 +142,25 @@ def build_overview_structure(self, module_tree: Dict[str, Any], module_path: Lis for child_name, child_info in module_info.items(): child_docs_path = self._resolve_child_docs_path(working_dir, child_name) if child_docs_path is not None: - child_info["docs"] = file_manager.load_text(child_docs_path) + child_info["docs_path"] = child_docs_path else: logger.warning(f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}") - child_info["docs"] = "" + child_info["docs_path"] = None return processed_module_tree + @classmethod + def _strip_components(cls, tree: Dict[str, Any]) -> None: + """Recursively drop ``components`` lists — they dominate the tree's + serialized size and add nothing to an overview prompt.""" + for module_info in tree.values(): + if not isinstance(module_info, dict): + continue + module_info.pop("components", None) + children = module_info.get("children") + if isinstance(children, dict): + cls._strip_components(children) + @staticmethod def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None: """Resolve the on-disk path for a child module's .md doc. @@ -274,17 +294,18 @@ async def generate_parent_module_docs(self, module_path: List[str], logger.info(f"✓ Parent docs already exists at {parent_docs_path}") return module_tree - # Create repo structure with 1-depth children docs and target indicator + # Create repo structure with 1-depth children doc paths and target indicator repo_structure = self.build_overview_structure(module_tree, module_path, working_dir) prompt = MODULE_OVERVIEW_PROMPT.format( module_name=module_name, - repo_structure=json.dumps(repo_structure, indent=4) + repo_structure=json.dumps(repo_structure, indent=2) ) if len(module_path) >= 1 else REPO_OVERVIEW_PROMPT.format( repo_name=module_name, - repo_structure=json.dumps(repo_structure, indent=4) + repo_structure=json.dumps(repo_structure, indent=2) ) - + logger.debug(f"Overview prompt for {module_name}: {len(prompt)} chars") + try: parent_docs = self.backend.complete(prompt) if not parent_docs: diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index 054d89e8..9eb17255 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -98,11 +98,13 @@ - The end-to-end architecture of the repository visualized by mermaid diagrams - The references to the core modules documentation -Provide `{repo_name}` repo structure and its core modules documentation: +Provide `{repo_name}` repo structure: {repo_structure} +The core modules' documentation is NOT inlined above. Each top-level module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null). + Please generate the overview of the `{repo_name}` repository in markdown format with the following structure: overview_content @@ -117,11 +119,13 @@ - The architecture of the module visualized by mermaid diagrams - The references to the core components documentation -Provide repo structure and core components documentation of the `{module_name}` module: +Provide repo structure of the `{module_name}` module (marked with `is_target_for_overview_generation`): {repo_structure} +The child modules' documentation is NOT inlined above. Each child of the target module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null). + Please generate the overview of the `{module_name}` module in markdown format with the following structure: overview_content diff --git a/pyproject.toml b/pyproject.toml index 25a0187c..93150ecf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,7 +58,7 @@ dependencies = [ "python-multipart>=0.0.20", "colorama>=0.4.6", "logfire>=4.1.0", - "coding-agent-wrapper>=0.1.2", + "coding-agent-wrapper @ git+https://github.com/anhnh2002/caw@fix/codex-exec-robustness", "mcp>=1.0.0" ] diff --git a/tests/test_overview_structure.py b/tests/test_overview_structure.py new file mode 100644 index 00000000..77f541cf --- /dev/null +++ b/tests/test_overview_structure.py @@ -0,0 +1,78 @@ +"""Tests for build_overview_structure's prompt-size discipline. + +Overview prompts used to inline the whole module tree (every module's +``components`` list) plus the full markdown of every child doc. On large +repos that blew past provider input caps — codex rejects any turn over +1,048,576 characters — and every parent/repo overview failed. The structure +must now strip ``components`` and reference children docs by absolute +``docs_path`` instead of inlining them. +""" + +from __future__ import annotations + +import json +import os + +from codewiki.src.be.documentation_generator import DocumentationGenerator + +CODEX_INPUT_CAP = 1_048_576 + + +def _generator() -> DocumentationGenerator: + # build_overview_structure touches neither config nor backend; skip the + # heavyweight __init__ (graph builder, backend resolution). + return DocumentationGenerator.__new__(DocumentationGenerator) + + +def _make_tree(n_modules: int, n_components: int) -> dict: + return { + f"module_{i}": { + "components": [f"src/file_{i}_{j}.c::func_{j}" for j in range(n_components)], + "children": { + f"module_{i}_child": { + "components": [f"src/file_{i}_{j}.c::helper_{j}" for j in range(n_components)], + "children": {}, + } + }, + } + for i in range(n_modules) + } + + +def test_components_are_stripped_at_every_depth(tmp_path): + tree = _make_tree(n_modules=3, n_components=5) + result = _generator().build_overview_structure(tree, [], str(tmp_path)) + assert "components" not in json.dumps(result) + # The original tree is untouched (deepcopy semantics). + assert tree["module_0"]["components"] + + +def test_children_docs_referenced_by_absolute_path_not_inlined(tmp_path): + tree = _make_tree(n_modules=2, n_components=1) + doc = tmp_path / "module_0.md" + doc.write_text("# module_0\n\nA very long body that must not appear in the prompt.") + + result = _generator().build_overview_structure(tree, [], str(tmp_path)) + + assert result["module_0"]["docs_path"] == str(doc) + assert os.path.isabs(result["module_0"]["docs_path"]) + assert result["module_1"]["docs_path"] is None # missing doc -> null, no crash + assert "must not appear" not in json.dumps(result) + + +def test_target_module_is_marked(tmp_path): + tree = _make_tree(n_modules=2, n_components=1) + result = _generator().build_overview_structure(tree, ["module_1"], str(tmp_path)) + assert result["module_1"]["is_target_for_overview_generation"] is True + # Children of the target get docs_path entries. + assert "docs_path" in result["module_1"]["children"]["module_1_child"] + + +def test_wazuh_scale_tree_stays_under_codex_input_cap(tmp_path): + # Modeled on the real failure: ~30 top-level modules, hundreds of + # components each, which serialized to ~2.1M chars before the fix. + tree = _make_tree(n_modules=30, n_components=600) + assert len(json.dumps(tree, indent=2)) > CODEX_INPUT_CAP # the old shape overflowed + + result = _generator().build_overview_structure(tree, [], str(tmp_path)) + assert len(json.dumps(result, indent=2)) < CODEX_INPUT_CAP / 10 From 72c357413745dd202d4f68b866312c887f54ba3c Mon Sep 17 00:00:00 2001 From: anhnh2002 Date: Sun, 16 Aug 2026 17:26:57 +0700 Subject: [PATCH 2/2] Satisfy CI lint on touched files The Lint job runs ruff (modern defaults) over every changed file, so the pre-existing violations in documentation_generator.py and prompt_template.py count against this PR. Mechanical cleanup only: import sorting, ruff format, Dict/List -> dict/list, explicit Optional, timezone-aware metadata timestamp, no mutable argument defaults, and dropping two duplicated dict keys (".hpp"/".jsx" in EXTENSION_TO_LANGUAGE). The two intentional blanket `except Exception` handlers are noqa'd with justification. Co-Authored-By: Claude Fable 5 --- codewiki/src/be/documentation_generator.py | 198 ++++++++++++--------- codewiki/src/be/prompt_template.py | 102 +++++++---- 2 files changed, 174 insertions(+), 126 deletions(-) diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py index 617292f1..654f0420 100644 --- a/codewiki/src/be/documentation_generator.py +++ b/codewiki/src/be/documentation_generator.py @@ -1,43 +1,43 @@ +import json import logging import os -import json -from typing import Dict, List, Any -from copy import deepcopy import traceback +from copy import deepcopy +from typing import Any # Configure logging and monitoring logger = logging.getLogger(__name__) # Local imports -from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder from codewiki.src.be.backend import LLMBackend, get_backend -from codewiki.src.be.prompt_template import ( - REPO_OVERVIEW_PROMPT, - MODULE_OVERVIEW_PROMPT, -) from codewiki.src.be.cluster_modules import ( cluster_modules, get_clustering_input_token_count, super_group_modules, ) -from codewiki.src.config import ( - Config, - FIRST_MODULE_TREE_FILENAME, - MODULE_TREE_FILENAME, - OVERVIEW_FILENAME -) +from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder from codewiki.src.be.module_naming import ( dedupe_module_tree_names, find_missing_module_docs, resolve_module_doc_path, ) +from codewiki.src.be.prompt_template import ( + MODULE_OVERVIEW_PROMPT, + REPO_OVERVIEW_PROMPT, +) +from codewiki.src.config import ( + FIRST_MODULE_TREE_FILENAME, + MODULE_TREE_FILENAME, + OVERVIEW_FILENAME, + Config, +) from codewiki.src.utils import file_manager class IncompleteDocumentationError(Exception): """Raised when generation finishes but some modules have no doc file on disk.""" - def __init__(self, missing_modules: List[str]): + def __init__(self, missing_modules: list[str]): self.missing_modules = missing_modules super().__init__( f"Documentation generation finished but {len(missing_modules)} module doc(s) " @@ -48,75 +48,82 @@ def __init__(self, missing_modules: List[str]): class DocumentationGenerator: """Main documentation generation orchestrator.""" - def __init__(self, config: Config, commit_id: str = None, backend: LLMBackend = None): + def __init__( + self, config: Config, commit_id: str | None = None, backend: LLMBackend | None = None + ): self.config = config self.commit_id = commit_id self.graph_builder = DependencyGraphBuilder(config) self.backend: LLMBackend = backend or get_backend(config) - - def create_documentation_metadata(self, working_dir: str, components: Dict[str, Any], num_leaf_nodes: int): + + def create_documentation_metadata( + self, working_dir: str, components: dict[str, Any], num_leaf_nodes: int + ): """Create a metadata file with documentation generation information.""" - from datetime import datetime - + from datetime import UTC, datetime + metadata = { "generation_info": { - "timestamp": datetime.now().isoformat(), + "timestamp": datetime.now(UTC).isoformat(), "main_model": self.config.main_model, "generator_version": "1.0.1", "repo_path": self.config.repo_path, - "commit_id": self.commit_id + "commit_id": self.commit_id, }, "statistics": { "total_components": len(components), "leaf_nodes": num_leaf_nodes, - "max_depth": self.config.max_depth + "max_depth": self.config.max_depth, }, - "files_generated": [ - "overview.md", - "module_tree.json", - "first_module_tree.json" - ] + "files_generated": ["overview.md", "module_tree.json", "first_module_tree.json"], } - + # Add generated markdown files to the metadata try: for file_path in os.listdir(working_dir): - if file_path.endswith('.md') and file_path not in metadata["files_generated"]: + if file_path.endswith(".md") and file_path not in metadata["files_generated"]: metadata["files_generated"].append(file_path) - except Exception as e: + except Exception as e: # noqa: BLE001 — metadata listing is best-effort logger.warning(f"Could not list generated files: {e}") - + metadata_path = os.path.join(working_dir, "metadata.json") file_manager.save_json(metadata, metadata_path) - - def get_processing_order(self, module_tree: Dict[str, Any], parent_path: List[str] = []) -> List[tuple[List[str], str]]: + def get_processing_order( + self, module_tree: dict[str, Any], parent_path: list[str] | None = None + ) -> list[tuple[list[str], str]]: """Get the processing order using topological sort (leaf modules first).""" + parent_path = parent_path or [] processing_order = [] - - def collect_modules(tree: Dict[str, Any], path: List[str]): + + def collect_modules(tree: dict[str, Any], path: list[str]): for module_name, module_info in tree.items(): current_path = path + [module_name] - + # If this module has children, process them first - if module_info.get("children") and isinstance(module_info["children"], dict) and module_info["children"]: + if ( + module_info.get("children") + and isinstance(module_info["children"], dict) + and module_info["children"] + ): collect_modules(module_info["children"], current_path) # Add this parent module after its children processing_order.append((current_path, module_name)) else: # This is a leaf module, add it immediately processing_order.append((current_path, module_name)) - + collect_modules(module_tree, parent_path) return processing_order - def is_leaf_module(self, module_info: Dict[str, Any]) -> bool: + def is_leaf_module(self, module_info: dict[str, Any]) -> bool: """Check if a module is a leaf module (has no children or empty children).""" children = module_info.get("children", {}) return not children or (isinstance(children, dict) and len(children) == 0) - def build_overview_structure(self, module_tree: Dict[str, Any], module_path: List[str], - working_dir: str) -> Dict[str, Any]: + def build_overview_structure( + self, module_tree: dict[str, Any], module_path: list[str], working_dir: str + ) -> dict[str, Any]: """Build structure for overview generation with 1-depth children doc paths and target indicator. Children docs are referenced by absolute file path (``docs_path``) @@ -144,13 +151,15 @@ def build_overview_structure(self, module_tree: Dict[str, Any], module_path: Lis if child_docs_path is not None: child_info["docs_path"] = child_docs_path else: - logger.warning(f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}") + logger.warning( + f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}" + ) child_info["docs_path"] = None return processed_module_tree @classmethod - def _strip_components(cls, tree: Dict[str, Any]) -> None: + def _strip_components(cls, tree: dict[str, Any]) -> None: """Recursively drop ``components`` lists — they dominate the tree's serialized size and add nothing to an overview prompt.""" for module_info in tree.values(): @@ -173,7 +182,7 @@ def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None: """ return resolve_module_doc_path(working_dir, child_name) - def validate_generated_docs(self, working_dir: str) -> List[str]: + def validate_generated_docs(self, working_dir: str) -> list[str]: """Check the final module tree against the docs on disk. Returns the names of modules whose .md file is missing (plus @@ -185,7 +194,9 @@ def validate_generated_docs(self, working_dir: str) -> List[str]: module_tree = file_manager.load_json(module_tree_path) return find_missing_module_docs(module_tree, working_dir) - async def generate_module_documentation(self, components: Dict[str, Any], leaf_nodes: List[str]) -> str: + async def generate_module_documentation( + self, components: dict[str, Any], leaf_nodes: list[str] + ) -> str: """Generate documentation for all modules using dynamic programming approach.""" # Prepare output directory working_dir = os.path.abspath(self.config.docs_dir) @@ -195,11 +206,10 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) first_module_tree = file_manager.load_json(first_module_tree_path) - + # Get processing order (leaf modules first) processing_order = self.get_processing_order(first_module_tree) - # Process modules in dependency order final_module_tree = module_tree processed_modules = set() @@ -209,19 +219,19 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n try: # Reload module tree to get latest hierarchical structure from sub-agent modifications module_tree = file_manager.load_json(module_tree_path) - + # Get the module info from the tree module_info = module_tree for path_part in module_path: module_info = module_info[path_part] if path_part != module_path[-1]: # Not the last part module_info = module_info.get("children", {}) - + # Skip if already processed module_key = "/".join(module_path) if module_key in processed_modules: continue - + # Process the module if self.is_leaf_module(module_info): logger.info(f"📄 Processing leaf module: {module_key}") @@ -237,21 +247,19 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n final_module_tree = await self.generate_parent_module_docs( module_path, working_dir ) - + processed_modules.add(module_key) - - except Exception as e: - logger.error(f"Failed to process module {module_key}: {str(e)}") + + except Exception as e: # noqa: BLE001 — one failed module must not abort the run + logger.error(f"Failed to process module {module_key}: {e!s}") logger.error(f"Traceback: {traceback.format_exc()}") continue # Generate repo overview - logger.info(f"📚 Generating repository overview") - final_module_tree = await self.generate_parent_module_docs( - [], working_dir - ) + logger.info("📚 Generating repository overview") + final_module_tree = await self.generate_parent_module_docs([], working_dir) else: - logger.info(f"Processing whole repo because repo can fit in the context window") + logger.info("Processing whole repo because repo can fit in the context window") repo_name = os.path.basename(os.path.normpath(self.config.repo_path)) final_module_tree = await self.backend.run_module_agent( module_name=repo_name, @@ -262,22 +270,29 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n ) # save final_module_tree to module_tree.json - file_manager.save_json(final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME)) + file_manager.save_json( + final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME) + ) # rename repo_name.md to overview.md repo_overview_path = os.path.join(working_dir, f"{repo_name}.md") if os.path.exists(repo_overview_path): os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME)) - + return working_dir - async def generate_parent_module_docs(self, module_path: List[str], - working_dir: str) -> Dict[str, Any]: + async def generate_parent_module_docs( + self, module_path: list[str], working_dir: str + ) -> dict[str, Any]: """Generate documentation for a parent module based on its children's documentation.""" - module_name = module_path[-1] if len(module_path) >= 1 else os.path.basename(os.path.normpath(self.config.repo_path)) + module_name = ( + module_path[-1] + if len(module_path) >= 1 + else os.path.basename(os.path.normpath(self.config.repo_path)) + ) logger.info(f"Generating parent documentation for: {module_name}") - + # Load module tree module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) module_tree = file_manager.load_json(module_tree_path) @@ -289,7 +304,10 @@ async def generate_parent_module_docs(self, module_path: List[str], return module_tree # check if parent docs already exists - parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md") + parent_docs_path = os.path.join( + working_dir, + f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md", + ) if os.path.exists(parent_docs_path): logger.info(f"✓ Parent docs already exists at {parent_docs_path}") return module_tree @@ -297,12 +315,14 @@ async def generate_parent_module_docs(self, module_path: List[str], # Create repo structure with 1-depth children doc paths and target indicator repo_structure = self.build_overview_structure(module_tree, module_path, working_dir) - prompt = MODULE_OVERVIEW_PROMPT.format( - module_name=module_name, - repo_structure=json.dumps(repo_structure, indent=2) - ) if len(module_path) >= 1 else REPO_OVERVIEW_PROMPT.format( - repo_name=module_name, - repo_structure=json.dumps(repo_structure, indent=2) + prompt = ( + MODULE_OVERVIEW_PROMPT.format( + module_name=module_name, repo_structure=json.dumps(repo_structure, indent=2) + ) + if len(module_path) >= 1 + else REPO_OVERVIEW_PROMPT.format( + repo_name=module_name, repo_structure=json.dumps(repo_structure, indent=2) + ) ) logger.debug(f"Overview prompt for {module_name}: {len(prompt)} chars") @@ -327,15 +347,15 @@ async def generate_parent_module_docs(self, module_path: List[str], ) parent_content = parent_docs.strip() file_manager.save_text(parent_content, parent_docs_path) - + logger.debug(f"Successfully generated parent documentation for: {module_name}") return module_tree - + except Exception as e: - logger.error(f"Error generating parent documentation for {module_name}: {str(e)}") + logger.error(f"Error generating parent documentation for {module_name}: {e!s}") logger.error(f"Traceback: {traceback.format_exc()}") raise - + async def run(self) -> None: """Run the complete documentation generation process using dynamic programming.""" try: @@ -345,13 +365,13 @@ async def run(self) -> None: logger.debug(f"Found {len(leaf_nodes)} leaf nodes") # logger.debug(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}") # exit() - + # Cluster modules working_dir = os.path.abspath(self.config.docs_dir) file_manager.ensure_directory(working_dir) first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME) module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME) - + # Check if module tree exists if os.path.exists(first_module_tree_path): logger.debug(f"Module tree found at {first_module_tree_path}") @@ -364,9 +384,7 @@ async def run(self) -> None: file_manager.save_json(module_tree, module_tree_path) else: logger.debug(f"Module tree not found at {module_tree_path}, clustering modules") - clustering_tokens = get_clustering_input_token_count( - leaf_nodes, components - ) + clustering_tokens = get_clustering_input_token_count(leaf_nodes, components) logger.info( "Preparing %d leaf nodes for module clustering (%d tokens, threshold %d)", len(leaf_nodes), @@ -395,7 +413,7 @@ async def run(self) -> None: module_tree = dedupe_module_tree_names(module_tree) file_manager.save_json(module_tree, first_module_tree_path) file_manager.save_json(module_tree, module_tree_path) - + if len(module_tree) == 0: logger.info( "Module clustering produced no top-level modules; continuing in " @@ -406,11 +424,11 @@ async def run(self) -> None: "Grouped components into %d top-level modules", len(module_tree), ) - + # Generate module documentation using dynamic programming approach # This processes leaf modules first, then parent modules working_dir = await self.generate_module_documentation(components, leaf_nodes) - + # Create documentation metadata self.create_documentation_metadata(working_dir, components, len(leaf_nodes)) @@ -422,11 +440,13 @@ async def run(self) -> None: logger.error(f"Module doc missing after generation: {module_name}.md") raise IncompleteDocumentationError(missing_docs) - logger.debug(f"Documentation generation completed successfully using dynamic programming!") - logger.debug(f"Processing order: leaf modules → parent modules → repository overview") + logger.debug( + "Documentation generation completed successfully using dynamic programming!" + ) + logger.debug("Processing order: leaf modules → parent modules → repository overview") logger.debug(f"Documentation saved to: {working_dir}") - + except Exception as e: - logger.error(f"Documentation generation failed: {str(e)}") + logger.error(f"Documentation generation failed: {e!s}") logger.error(f"Traceback: {traceback.format_exc()}") raise diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py index 9eb17255..49783052 100644 --- a/codewiki/src/be/prompt_template.py +++ b/codewiki/src/be/prompt_template.py @@ -257,7 +257,8 @@ Reasoning at first, then return the list of relative paths in JSON format. """ -from typing import Dict, Any +from typing import Any + from codewiki.src.utils import file_manager EXTENSION_TO_LANGUAGE = { @@ -275,37 +276,40 @@ ".hpp": "cpp", ".tsx": "typescript", ".cc": "cpp", - ".hpp": "cpp", ".cxx": "cpp", ".jsx": "javascript", ".mjs": "javascript", ".cjs": "javascript", - ".jsx": "javascript", ".cs": "csharp", ".kt": "kotlin", ".kts": "kotlin", ".php": "php", ".phtml": "php", - ".inc": "php" + ".inc": "php", } -def format_user_prompt(module_name: str, core_component_ids: list[str], components: Dict[str, Any], module_tree: dict[str, any]) -> str: +def format_user_prompt( + module_name: str, + core_component_ids: list[str], + components: dict[str, Any], + module_tree: dict[str, any], +) -> str: """ Format the user prompt with module name and organized core component codes. - + Args: module_name: Name of the module to document core_component_ids: List of component IDs to include components: Dictionary mapping component IDs to CodeComponent objects - + Returns: Formatted user prompt string """ # format module tree lines = [] - + def _format_module_tree(module_tree: dict[str, any], indent: int = 0): for key, value in module_tree.items(): if key == module_name: @@ -315,8 +319,9 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): # Group components by file from collections import defaultdict + by_file = defaultdict(list) - for c in value['components']: + for c in value["components"]: if "::" in c: fpath, name = c.split("::", 1) by_file[fpath].append(name) @@ -351,46 +356,60 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): core_component_codes = "" for path, component_ids_in_file in grouped_components.items(): core_component_codes += f"# File: {path}\n\n" - core_component_codes += f"## Core Components in this file:\n" - + core_component_codes += "## Core Components in this file:\n" + for component_id in component_ids_in_file: core_component_codes += f"- {component_id}\n" - - core_component_codes += f"\n## File Content:\n```{EXTENSION_TO_LANGUAGE['.'+path.split('.')[-1]]}\n" - + + core_component_codes += ( + f"\n## File Content:\n```{EXTENSION_TO_LANGUAGE['.' + path.split('.')[-1]]}\n" + ) + # Read content of the file using the first component's file path try: - core_component_codes += file_manager.load_text(components[component_ids_in_file[0]].file_path) - except (FileNotFoundError, IOError) as e: + core_component_codes += file_manager.load_text( + components[component_ids_in_file[0]].file_path + ) + except (OSError, FileNotFoundError) as e: core_component_codes += f"# Error reading file: {e}\n" - + core_component_codes += "```\n\n" - - return USER_PROMPT.format(module_name=module_name, formatted_core_component_codes=core_component_codes, module_tree=formatted_module_tree) + return USER_PROMPT.format( + module_name=module_name, + formatted_core_component_codes=core_component_codes, + module_tree=formatted_module_tree, + ) -def format_cluster_prompt(potential_core_components: str, module_tree: dict[str, any] = {}, module_name: str = None) -> str: +def format_cluster_prompt( + potential_core_components: str, + module_tree: dict[str, any] | None = None, + module_name: str | None = None, +) -> str: """ Format the cluster prompt with potential core components and module tree. """ + if module_tree is None: + module_tree = {} # format module tree lines = [] # print(f"Module tree:\n{json.dumps(module_tree, indent=2)}") - + def _format_module_tree(module_tree: dict[str, any], indent: int = 0): for key, value in module_tree.items(): if key == module_name: lines.append(f"{' ' * indent}{key} (current module)") else: lines.append(f"{' ' * indent}{key}") - + # Group components by file from collections import defaultdict + by_file = defaultdict(list) - for c in value['components']: + for c in value["components"]: if "::" in c: fpath, name = c.split("::", 1) by_file[fpath].append(name) @@ -402,21 +421,28 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0): else: lines.append(f"{' ' * (indent + 1)} {', '.join(names)}") - if ("children" in value) and isinstance(value["children"], dict) and len(value["children"]) > 0: + if ( + ("children" in value) + and isinstance(value["children"], dict) + and len(value["children"]) > 0 + ): lines.append(f"{' ' * (indent + 1)} Children:") _format_module_tree(value["children"], indent + 2) - + _format_module_tree(module_tree, 0) formatted_module_tree = "\n".join(lines) - if module_tree == {}: return CLUSTER_REPO_PROMPT.format(potential_core_components=potential_core_components) else: - return CLUSTER_MODULE_PROMPT.format(potential_core_components=potential_core_components, module_tree=formatted_module_tree, module_name=module_name) + return CLUSTER_MODULE_PROMPT.format( + potential_core_components=potential_core_components, + module_tree=formatted_module_tree, + module_name=module_name, + ) -def format_super_group_prompt(module_tree: Dict[str, Any]) -> str: +def format_super_group_prompt(module_tree: dict[str, Any]) -> str: """ Format the super-grouping prompt with the flat top-level modules of a tree. """ @@ -429,37 +455,39 @@ def format_super_group_prompt(module_tree: Dict[str, Any]) -> str: return SUPER_GROUP_PROMPT.format(formatted_modules="\n".join(lines)) -def format_system_prompt(module_name: str, custom_instructions: str = None) -> str: +def format_system_prompt(module_name: str, custom_instructions: str | None = None) -> str: """ Format the system prompt with module name and optional custom instructions. - + Args: module_name: Name of the module to document custom_instructions: Optional custom instructions to append - + Returns: Formatted system prompt string """ custom_section = "" if custom_instructions: custom_section = f"\n\n\n{custom_instructions}\n" - + return SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() -def format_leaf_system_prompt(module_name: str, custom_instructions: str = None) -> str: +def format_leaf_system_prompt(module_name: str, custom_instructions: str | None = None) -> str: """ Format the leaf system prompt with module name and optional custom instructions. - + Args: module_name: Name of the module to document custom_instructions: Optional custom instructions to append - + Returns: Formatted leaf system prompt string """ custom_section = "" if custom_instructions: custom_section = f"\n\n\n{custom_instructions}\n" - - return LEAF_SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip() \ No newline at end of file + + return LEAF_SYSTEM_PROMPT.format( + module_name=module_name, custom_instructions=custom_section + ).strip()