diff --git a/codewiki/src/be/documentation_generator.py b/codewiki/src/be/documentation_generator.py
index a91ee426..654f0420 100644
--- a/codewiki/src/be/documentation_generator.py
+++ b/codewiki/src/be/documentation_generator.py
@@ -1,43 +1,43 @@
+import json
import logging
import os
-import json
-from typing import Dict, List, Any
-from copy import deepcopy
import traceback
+from copy import deepcopy
+from typing import Any
# Configure logging and monitoring
logger = logging.getLogger(__name__)
# Local imports
-from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder
from codewiki.src.be.backend import LLMBackend, get_backend
-from codewiki.src.be.prompt_template import (
- REPO_OVERVIEW_PROMPT,
- MODULE_OVERVIEW_PROMPT,
-)
from codewiki.src.be.cluster_modules import (
cluster_modules,
get_clustering_input_token_count,
super_group_modules,
)
-from codewiki.src.config import (
- Config,
- FIRST_MODULE_TREE_FILENAME,
- MODULE_TREE_FILENAME,
- OVERVIEW_FILENAME
-)
+from codewiki.src.be.dependency_analyzer import DependencyGraphBuilder
from codewiki.src.be.module_naming import (
dedupe_module_tree_names,
find_missing_module_docs,
resolve_module_doc_path,
)
+from codewiki.src.be.prompt_template import (
+ MODULE_OVERVIEW_PROMPT,
+ REPO_OVERVIEW_PROMPT,
+)
+from codewiki.src.config import (
+ FIRST_MODULE_TREE_FILENAME,
+ MODULE_TREE_FILENAME,
+ OVERVIEW_FILENAME,
+ Config,
+)
from codewiki.src.utils import file_manager
class IncompleteDocumentationError(Exception):
"""Raised when generation finishes but some modules have no doc file on disk."""
- def __init__(self, missing_modules: List[str]):
+ def __init__(self, missing_modules: list[str]):
self.missing_modules = missing_modules
super().__init__(
f"Documentation generation finished but {len(missing_modules)} module doc(s) "
@@ -48,78 +48,93 @@ def __init__(self, missing_modules: List[str]):
class DocumentationGenerator:
"""Main documentation generation orchestrator."""
- def __init__(self, config: Config, commit_id: str = None, backend: LLMBackend = None):
+ def __init__(
+ self, config: Config, commit_id: str | None = None, backend: LLMBackend | None = None
+ ):
self.config = config
self.commit_id = commit_id
self.graph_builder = DependencyGraphBuilder(config)
self.backend: LLMBackend = backend or get_backend(config)
-
- def create_documentation_metadata(self, working_dir: str, components: Dict[str, Any], num_leaf_nodes: int):
+
+ def create_documentation_metadata(
+ self, working_dir: str, components: dict[str, Any], num_leaf_nodes: int
+ ):
"""Create a metadata file with documentation generation information."""
- from datetime import datetime
-
+ from datetime import UTC, datetime
+
metadata = {
"generation_info": {
- "timestamp": datetime.now().isoformat(),
+ "timestamp": datetime.now(UTC).isoformat(),
"main_model": self.config.main_model,
"generator_version": "1.0.1",
"repo_path": self.config.repo_path,
- "commit_id": self.commit_id
+ "commit_id": self.commit_id,
},
"statistics": {
"total_components": len(components),
"leaf_nodes": num_leaf_nodes,
- "max_depth": self.config.max_depth
+ "max_depth": self.config.max_depth,
},
- "files_generated": [
- "overview.md",
- "module_tree.json",
- "first_module_tree.json"
- ]
+ "files_generated": ["overview.md", "module_tree.json", "first_module_tree.json"],
}
-
+
# Add generated markdown files to the metadata
try:
for file_path in os.listdir(working_dir):
- if file_path.endswith('.md') and file_path not in metadata["files_generated"]:
+ if file_path.endswith(".md") and file_path not in metadata["files_generated"]:
metadata["files_generated"].append(file_path)
- except Exception as e:
+ except Exception as e: # noqa: BLE001 — metadata listing is best-effort
logger.warning(f"Could not list generated files: {e}")
-
+
metadata_path = os.path.join(working_dir, "metadata.json")
file_manager.save_json(metadata, metadata_path)
-
- def get_processing_order(self, module_tree: Dict[str, Any], parent_path: List[str] = []) -> List[tuple[List[str], str]]:
+ def get_processing_order(
+ self, module_tree: dict[str, Any], parent_path: list[str] | None = None
+ ) -> list[tuple[list[str], str]]:
"""Get the processing order using topological sort (leaf modules first)."""
+ parent_path = parent_path or []
processing_order = []
-
- def collect_modules(tree: Dict[str, Any], path: List[str]):
+
+ def collect_modules(tree: dict[str, Any], path: list[str]):
for module_name, module_info in tree.items():
current_path = path + [module_name]
-
+
# If this module has children, process them first
- if module_info.get("children") and isinstance(module_info["children"], dict) and module_info["children"]:
+ if (
+ module_info.get("children")
+ and isinstance(module_info["children"], dict)
+ and module_info["children"]
+ ):
collect_modules(module_info["children"], current_path)
# Add this parent module after its children
processing_order.append((current_path, module_name))
else:
# This is a leaf module, add it immediately
processing_order.append((current_path, module_name))
-
+
collect_modules(module_tree, parent_path)
return processing_order
- def is_leaf_module(self, module_info: Dict[str, Any]) -> bool:
+ def is_leaf_module(self, module_info: dict[str, Any]) -> bool:
"""Check if a module is a leaf module (has no children or empty children)."""
children = module_info.get("children", {})
return not children or (isinstance(children, dict) and len(children) == 0)
- def build_overview_structure(self, module_tree: Dict[str, Any], module_path: List[str],
- working_dir: str) -> Dict[str, Any]:
- """Build structure for overview generation with 1-depth children docs and target indicator."""
-
+ def build_overview_structure(
+ self, module_tree: dict[str, Any], module_path: list[str], working_dir: str
+ ) -> dict[str, Any]:
+ """Build structure for overview generation with 1-depth children doc paths and target indicator.
+
+ Children docs are referenced by absolute file path (``docs_path``)
+ rather than inlined, and ``components`` lists are stripped: inlining
+ the full tree plus docs blew past provider input caps (codex rejects
+ turns over 1,048,576 chars) on large repos. The overview agent reads
+ the referenced files itself.
+ """
+
processed_module_tree = deepcopy(module_tree)
+ self._strip_components(processed_module_tree)
module_info = processed_module_tree
for path_part in module_path:
module_info = module_info[path_part]
@@ -134,13 +149,27 @@ def build_overview_structure(self, module_tree: Dict[str, Any], module_path: Lis
for child_name, child_info in module_info.items():
child_docs_path = self._resolve_child_docs_path(working_dir, child_name)
if child_docs_path is not None:
- child_info["docs"] = file_manager.load_text(child_docs_path)
+ child_info["docs_path"] = child_docs_path
else:
- logger.warning(f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}")
- child_info["docs"] = ""
+ logger.warning(
+ f"Module docs not found at {os.path.join(working_dir, f'{child_name}.md')}"
+ )
+ child_info["docs_path"] = None
return processed_module_tree
+ @classmethod
+ def _strip_components(cls, tree: dict[str, Any]) -> None:
+ """Recursively drop ``components`` lists — they dominate the tree's
+ serialized size and add nothing to an overview prompt."""
+ for module_info in tree.values():
+ if not isinstance(module_info, dict):
+ continue
+ module_info.pop("components", None)
+ children = module_info.get("children")
+ if isinstance(children, dict):
+ cls._strip_components(children)
+
@staticmethod
def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None:
"""Resolve the on-disk path for a child module's .md doc.
@@ -153,7 +182,7 @@ def _resolve_child_docs_path(working_dir: str, child_name: str) -> str | None:
"""
return resolve_module_doc_path(working_dir, child_name)
- def validate_generated_docs(self, working_dir: str) -> List[str]:
+ def validate_generated_docs(self, working_dir: str) -> list[str]:
"""Check the final module tree against the docs on disk.
Returns the names of modules whose .md file is missing (plus
@@ -165,7 +194,9 @@ def validate_generated_docs(self, working_dir: str) -> List[str]:
module_tree = file_manager.load_json(module_tree_path)
return find_missing_module_docs(module_tree, working_dir)
- async def generate_module_documentation(self, components: Dict[str, Any], leaf_nodes: List[str]) -> str:
+ async def generate_module_documentation(
+ self, components: dict[str, Any], leaf_nodes: list[str]
+ ) -> str:
"""Generate documentation for all modules using dynamic programming approach."""
# Prepare output directory
working_dir = os.path.abspath(self.config.docs_dir)
@@ -175,11 +206,10 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n
first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME)
module_tree = file_manager.load_json(module_tree_path)
first_module_tree = file_manager.load_json(first_module_tree_path)
-
+
# Get processing order (leaf modules first)
processing_order = self.get_processing_order(first_module_tree)
-
# Process modules in dependency order
final_module_tree = module_tree
processed_modules = set()
@@ -189,19 +219,19 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n
try:
# Reload module tree to get latest hierarchical structure from sub-agent modifications
module_tree = file_manager.load_json(module_tree_path)
-
+
# Get the module info from the tree
module_info = module_tree
for path_part in module_path:
module_info = module_info[path_part]
if path_part != module_path[-1]: # Not the last part
module_info = module_info.get("children", {})
-
+
# Skip if already processed
module_key = "/".join(module_path)
if module_key in processed_modules:
continue
-
+
# Process the module
if self.is_leaf_module(module_info):
logger.info(f"📄 Processing leaf module: {module_key}")
@@ -217,21 +247,19 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n
final_module_tree = await self.generate_parent_module_docs(
module_path, working_dir
)
-
+
processed_modules.add(module_key)
-
- except Exception as e:
- logger.error(f"Failed to process module {module_key}: {str(e)}")
+
+ except Exception as e: # noqa: BLE001 — one failed module must not abort the run
+ logger.error(f"Failed to process module {module_key}: {e!s}")
logger.error(f"Traceback: {traceback.format_exc()}")
continue
# Generate repo overview
- logger.info(f"📚 Generating repository overview")
- final_module_tree = await self.generate_parent_module_docs(
- [], working_dir
- )
+ logger.info("📚 Generating repository overview")
+ final_module_tree = await self.generate_parent_module_docs([], working_dir)
else:
- logger.info(f"Processing whole repo because repo can fit in the context window")
+ logger.info("Processing whole repo because repo can fit in the context window")
repo_name = os.path.basename(os.path.normpath(self.config.repo_path))
final_module_tree = await self.backend.run_module_agent(
module_name=repo_name,
@@ -242,22 +270,29 @@ async def generate_module_documentation(self, components: Dict[str, Any], leaf_n
)
# save final_module_tree to module_tree.json
- file_manager.save_json(final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME))
+ file_manager.save_json(
+ final_module_tree, os.path.join(working_dir, MODULE_TREE_FILENAME)
+ )
# rename repo_name.md to overview.md
repo_overview_path = os.path.join(working_dir, f"{repo_name}.md")
if os.path.exists(repo_overview_path):
os.rename(repo_overview_path, os.path.join(working_dir, OVERVIEW_FILENAME))
-
+
return working_dir
- async def generate_parent_module_docs(self, module_path: List[str],
- working_dir: str) -> Dict[str, Any]:
+ async def generate_parent_module_docs(
+ self, module_path: list[str], working_dir: str
+ ) -> dict[str, Any]:
"""Generate documentation for a parent module based on its children's documentation."""
- module_name = module_path[-1] if len(module_path) >= 1 else os.path.basename(os.path.normpath(self.config.repo_path))
+ module_name = (
+ module_path[-1]
+ if len(module_path) >= 1
+ else os.path.basename(os.path.normpath(self.config.repo_path))
+ )
logger.info(f"Generating parent documentation for: {module_name}")
-
+
# Load module tree
module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME)
module_tree = file_manager.load_json(module_tree_path)
@@ -269,22 +304,28 @@ async def generate_parent_module_docs(self, module_path: List[str],
return module_tree
# check if parent docs already exists
- parent_docs_path = os.path.join(working_dir, f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md")
+ parent_docs_path = os.path.join(
+ working_dir,
+ f"{module_name if len(module_path) >= 1 else OVERVIEW_FILENAME.replace('.md', '')}.md",
+ )
if os.path.exists(parent_docs_path):
logger.info(f"✓ Parent docs already exists at {parent_docs_path}")
return module_tree
- # Create repo structure with 1-depth children docs and target indicator
+ # Create repo structure with 1-depth children doc paths and target indicator
repo_structure = self.build_overview_structure(module_tree, module_path, working_dir)
- prompt = MODULE_OVERVIEW_PROMPT.format(
- module_name=module_name,
- repo_structure=json.dumps(repo_structure, indent=4)
- ) if len(module_path) >= 1 else REPO_OVERVIEW_PROMPT.format(
- repo_name=module_name,
- repo_structure=json.dumps(repo_structure, indent=4)
+ prompt = (
+ MODULE_OVERVIEW_PROMPT.format(
+ module_name=module_name, repo_structure=json.dumps(repo_structure, indent=2)
+ )
+ if len(module_path) >= 1
+ else REPO_OVERVIEW_PROMPT.format(
+ repo_name=module_name, repo_structure=json.dumps(repo_structure, indent=2)
+ )
)
-
+ logger.debug(f"Overview prompt for {module_name}: {len(prompt)} chars")
+
try:
parent_docs = self.backend.complete(prompt)
if not parent_docs:
@@ -306,15 +347,15 @@ async def generate_parent_module_docs(self, module_path: List[str],
)
parent_content = parent_docs.strip()
file_manager.save_text(parent_content, parent_docs_path)
-
+
logger.debug(f"Successfully generated parent documentation for: {module_name}")
return module_tree
-
+
except Exception as e:
- logger.error(f"Error generating parent documentation for {module_name}: {str(e)}")
+ logger.error(f"Error generating parent documentation for {module_name}: {e!s}")
logger.error(f"Traceback: {traceback.format_exc()}")
raise
-
+
async def run(self) -> None:
"""Run the complete documentation generation process using dynamic programming."""
try:
@@ -324,13 +365,13 @@ async def run(self) -> None:
logger.debug(f"Found {len(leaf_nodes)} leaf nodes")
# logger.debug(f"Leaf nodes:\n{'\n'.join(sorted(leaf_nodes)[:200])}")
# exit()
-
+
# Cluster modules
working_dir = os.path.abspath(self.config.docs_dir)
file_manager.ensure_directory(working_dir)
first_module_tree_path = os.path.join(working_dir, FIRST_MODULE_TREE_FILENAME)
module_tree_path = os.path.join(working_dir, MODULE_TREE_FILENAME)
-
+
# Check if module tree exists
if os.path.exists(first_module_tree_path):
logger.debug(f"Module tree found at {first_module_tree_path}")
@@ -343,9 +384,7 @@ async def run(self) -> None:
file_manager.save_json(module_tree, module_tree_path)
else:
logger.debug(f"Module tree not found at {module_tree_path}, clustering modules")
- clustering_tokens = get_clustering_input_token_count(
- leaf_nodes, components
- )
+ clustering_tokens = get_clustering_input_token_count(leaf_nodes, components)
logger.info(
"Preparing %d leaf nodes for module clustering (%d tokens, threshold %d)",
len(leaf_nodes),
@@ -374,7 +413,7 @@ async def run(self) -> None:
module_tree = dedupe_module_tree_names(module_tree)
file_manager.save_json(module_tree, first_module_tree_path)
file_manager.save_json(module_tree, module_tree_path)
-
+
if len(module_tree) == 0:
logger.info(
"Module clustering produced no top-level modules; continuing in "
@@ -385,11 +424,11 @@ async def run(self) -> None:
"Grouped components into %d top-level modules",
len(module_tree),
)
-
+
# Generate module documentation using dynamic programming approach
# This processes leaf modules first, then parent modules
working_dir = await self.generate_module_documentation(components, leaf_nodes)
-
+
# Create documentation metadata
self.create_documentation_metadata(working_dir, components, len(leaf_nodes))
@@ -401,11 +440,13 @@ async def run(self) -> None:
logger.error(f"Module doc missing after generation: {module_name}.md")
raise IncompleteDocumentationError(missing_docs)
- logger.debug(f"Documentation generation completed successfully using dynamic programming!")
- logger.debug(f"Processing order: leaf modules → parent modules → repository overview")
+ logger.debug(
+ "Documentation generation completed successfully using dynamic programming!"
+ )
+ logger.debug("Processing order: leaf modules → parent modules → repository overview")
logger.debug(f"Documentation saved to: {working_dir}")
-
+
except Exception as e:
- logger.error(f"Documentation generation failed: {str(e)}")
+ logger.error(f"Documentation generation failed: {e!s}")
logger.error(f"Traceback: {traceback.format_exc()}")
raise
diff --git a/codewiki/src/be/prompt_template.py b/codewiki/src/be/prompt_template.py
index 054d89e8..49783052 100644
--- a/codewiki/src/be/prompt_template.py
+++ b/codewiki/src/be/prompt_template.py
@@ -98,11 +98,13 @@
- The end-to-end architecture of the repository visualized by mermaid diagrams
- The references to the core modules documentation
-Provide `{repo_name}` repo structure and its core modules documentation:
+Provide `{repo_name}` repo structure:
{repo_structure}
+The core modules' documentation is NOT inlined above. Each top-level module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null).
+
Please generate the overview of the `{repo_name}` repository in markdown format with the following structure:
overview_content
@@ -117,11 +119,13 @@
- The architecture of the module visualized by mermaid diagrams
- The references to the core components documentation
-Provide repo structure and core components documentation of the `{module_name}` module:
+Provide repo structure of the `{module_name}` module (marked with `is_target_for_overview_generation`):
{repo_structure}
+The child modules' documentation is NOT inlined above. Each child of the target module carries a `docs_path` field with the absolute path to its documentation file — read those files with your file-reading tools before writing the overview (skip entries whose `docs_path` is null).
+
Please generate the overview of the `{module_name}` module in markdown format with the following structure:
overview_content
@@ -253,7 +257,8 @@
Reasoning at first, then return the list of relative paths in JSON format.
"""
-from typing import Dict, Any
+from typing import Any
+
from codewiki.src.utils import file_manager
EXTENSION_TO_LANGUAGE = {
@@ -271,37 +276,40 @@
".hpp": "cpp",
".tsx": "typescript",
".cc": "cpp",
- ".hpp": "cpp",
".cxx": "cpp",
".jsx": "javascript",
".mjs": "javascript",
".cjs": "javascript",
- ".jsx": "javascript",
".cs": "csharp",
".kt": "kotlin",
".kts": "kotlin",
".php": "php",
".phtml": "php",
- ".inc": "php"
+ ".inc": "php",
}
-def format_user_prompt(module_name: str, core_component_ids: list[str], components: Dict[str, Any], module_tree: dict[str, any]) -> str:
+def format_user_prompt(
+ module_name: str,
+ core_component_ids: list[str],
+ components: dict[str, Any],
+ module_tree: dict[str, any],
+) -> str:
"""
Format the user prompt with module name and organized core component codes.
-
+
Args:
module_name: Name of the module to document
core_component_ids: List of component IDs to include
components: Dictionary mapping component IDs to CodeComponent objects
-
+
Returns:
Formatted user prompt string
"""
# format module tree
lines = []
-
+
def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
for key, value in module_tree.items():
if key == module_name:
@@ -311,8 +319,9 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
# Group components by file
from collections import defaultdict
+
by_file = defaultdict(list)
- for c in value['components']:
+ for c in value["components"]:
if "::" in c:
fpath, name = c.split("::", 1)
by_file[fpath].append(name)
@@ -347,46 +356,60 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
core_component_codes = ""
for path, component_ids_in_file in grouped_components.items():
core_component_codes += f"# File: {path}\n\n"
- core_component_codes += f"## Core Components in this file:\n"
-
+ core_component_codes += "## Core Components in this file:\n"
+
for component_id in component_ids_in_file:
core_component_codes += f"- {component_id}\n"
-
- core_component_codes += f"\n## File Content:\n```{EXTENSION_TO_LANGUAGE['.'+path.split('.')[-1]]}\n"
-
+
+ core_component_codes += (
+ f"\n## File Content:\n```{EXTENSION_TO_LANGUAGE['.' + path.split('.')[-1]]}\n"
+ )
+
# Read content of the file using the first component's file path
try:
- core_component_codes += file_manager.load_text(components[component_ids_in_file[0]].file_path)
- except (FileNotFoundError, IOError) as e:
+ core_component_codes += file_manager.load_text(
+ components[component_ids_in_file[0]].file_path
+ )
+ except (OSError, FileNotFoundError) as e:
core_component_codes += f"# Error reading file: {e}\n"
-
+
core_component_codes += "```\n\n"
-
- return USER_PROMPT.format(module_name=module_name, formatted_core_component_codes=core_component_codes, module_tree=formatted_module_tree)
+ return USER_PROMPT.format(
+ module_name=module_name,
+ formatted_core_component_codes=core_component_codes,
+ module_tree=formatted_module_tree,
+ )
-def format_cluster_prompt(potential_core_components: str, module_tree: dict[str, any] = {}, module_name: str = None) -> str:
+def format_cluster_prompt(
+ potential_core_components: str,
+ module_tree: dict[str, any] | None = None,
+ module_name: str | None = None,
+) -> str:
"""
Format the cluster prompt with potential core components and module tree.
"""
+ if module_tree is None:
+ module_tree = {}
# format module tree
lines = []
# print(f"Module tree:\n{json.dumps(module_tree, indent=2)}")
-
+
def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
for key, value in module_tree.items():
if key == module_name:
lines.append(f"{' ' * indent}{key} (current module)")
else:
lines.append(f"{' ' * indent}{key}")
-
+
# Group components by file
from collections import defaultdict
+
by_file = defaultdict(list)
- for c in value['components']:
+ for c in value["components"]:
if "::" in c:
fpath, name = c.split("::", 1)
by_file[fpath].append(name)
@@ -398,21 +421,28 @@ def _format_module_tree(module_tree: dict[str, any], indent: int = 0):
else:
lines.append(f"{' ' * (indent + 1)} {', '.join(names)}")
- if ("children" in value) and isinstance(value["children"], dict) and len(value["children"]) > 0:
+ if (
+ ("children" in value)
+ and isinstance(value["children"], dict)
+ and len(value["children"]) > 0
+ ):
lines.append(f"{' ' * (indent + 1)} Children:")
_format_module_tree(value["children"], indent + 2)
-
+
_format_module_tree(module_tree, 0)
formatted_module_tree = "\n".join(lines)
-
if module_tree == {}:
return CLUSTER_REPO_PROMPT.format(potential_core_components=potential_core_components)
else:
- return CLUSTER_MODULE_PROMPT.format(potential_core_components=potential_core_components, module_tree=formatted_module_tree, module_name=module_name)
+ return CLUSTER_MODULE_PROMPT.format(
+ potential_core_components=potential_core_components,
+ module_tree=formatted_module_tree,
+ module_name=module_name,
+ )
-def format_super_group_prompt(module_tree: Dict[str, Any]) -> str:
+def format_super_group_prompt(module_tree: dict[str, Any]) -> str:
"""
Format the super-grouping prompt with the flat top-level modules of a tree.
"""
@@ -425,37 +455,39 @@ def format_super_group_prompt(module_tree: Dict[str, Any]) -> str:
return SUPER_GROUP_PROMPT.format(formatted_modules="\n".join(lines))
-def format_system_prompt(module_name: str, custom_instructions: str = None) -> str:
+def format_system_prompt(module_name: str, custom_instructions: str | None = None) -> str:
"""
Format the system prompt with module name and optional custom instructions.
-
+
Args:
module_name: Name of the module to document
custom_instructions: Optional custom instructions to append
-
+
Returns:
Formatted system prompt string
"""
custom_section = ""
if custom_instructions:
custom_section = f"\n\n\n{custom_instructions}\n"
-
+
return SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip()
-def format_leaf_system_prompt(module_name: str, custom_instructions: str = None) -> str:
+def format_leaf_system_prompt(module_name: str, custom_instructions: str | None = None) -> str:
"""
Format the leaf system prompt with module name and optional custom instructions.
-
+
Args:
module_name: Name of the module to document
custom_instructions: Optional custom instructions to append
-
+
Returns:
Formatted leaf system prompt string
"""
custom_section = ""
if custom_instructions:
custom_section = f"\n\n\n{custom_instructions}\n"
-
- return LEAF_SYSTEM_PROMPT.format(module_name=module_name, custom_instructions=custom_section).strip()
\ No newline at end of file
+
+ return LEAF_SYSTEM_PROMPT.format(
+ module_name=module_name, custom_instructions=custom_section
+ ).strip()
diff --git a/pyproject.toml b/pyproject.toml
index 25a0187c..93150ecf 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,7 @@ dependencies = [
"python-multipart>=0.0.20",
"colorama>=0.4.6",
"logfire>=4.1.0",
- "coding-agent-wrapper>=0.1.2",
+ "coding-agent-wrapper @ git+https://github.com/anhnh2002/caw@fix/codex-exec-robustness",
"mcp>=1.0.0"
]
diff --git a/tests/test_overview_structure.py b/tests/test_overview_structure.py
new file mode 100644
index 00000000..77f541cf
--- /dev/null
+++ b/tests/test_overview_structure.py
@@ -0,0 +1,78 @@
+"""Tests for build_overview_structure's prompt-size discipline.
+
+Overview prompts used to inline the whole module tree (every module's
+``components`` list) plus the full markdown of every child doc. On large
+repos that blew past provider input caps — codex rejects any turn over
+1,048,576 characters — and every parent/repo overview failed. The structure
+must now strip ``components`` and reference children docs by absolute
+``docs_path`` instead of inlining them.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from codewiki.src.be.documentation_generator import DocumentationGenerator
+
+CODEX_INPUT_CAP = 1_048_576
+
+
+def _generator() -> DocumentationGenerator:
+ # build_overview_structure touches neither config nor backend; skip the
+ # heavyweight __init__ (graph builder, backend resolution).
+ return DocumentationGenerator.__new__(DocumentationGenerator)
+
+
+def _make_tree(n_modules: int, n_components: int) -> dict:
+ return {
+ f"module_{i}": {
+ "components": [f"src/file_{i}_{j}.c::func_{j}" for j in range(n_components)],
+ "children": {
+ f"module_{i}_child": {
+ "components": [f"src/file_{i}_{j}.c::helper_{j}" for j in range(n_components)],
+ "children": {},
+ }
+ },
+ }
+ for i in range(n_modules)
+ }
+
+
+def test_components_are_stripped_at_every_depth(tmp_path):
+ tree = _make_tree(n_modules=3, n_components=5)
+ result = _generator().build_overview_structure(tree, [], str(tmp_path))
+ assert "components" not in json.dumps(result)
+ # The original tree is untouched (deepcopy semantics).
+ assert tree["module_0"]["components"]
+
+
+def test_children_docs_referenced_by_absolute_path_not_inlined(tmp_path):
+ tree = _make_tree(n_modules=2, n_components=1)
+ doc = tmp_path / "module_0.md"
+ doc.write_text("# module_0\n\nA very long body that must not appear in the prompt.")
+
+ result = _generator().build_overview_structure(tree, [], str(tmp_path))
+
+ assert result["module_0"]["docs_path"] == str(doc)
+ assert os.path.isabs(result["module_0"]["docs_path"])
+ assert result["module_1"]["docs_path"] is None # missing doc -> null, no crash
+ assert "must not appear" not in json.dumps(result)
+
+
+def test_target_module_is_marked(tmp_path):
+ tree = _make_tree(n_modules=2, n_components=1)
+ result = _generator().build_overview_structure(tree, ["module_1"], str(tmp_path))
+ assert result["module_1"]["is_target_for_overview_generation"] is True
+ # Children of the target get docs_path entries.
+ assert "docs_path" in result["module_1"]["children"]["module_1_child"]
+
+
+def test_wazuh_scale_tree_stays_under_codex_input_cap(tmp_path):
+ # Modeled on the real failure: ~30 top-level modules, hundreds of
+ # components each, which serialized to ~2.1M chars before the fix.
+ tree = _make_tree(n_modules=30, n_components=600)
+ assert len(json.dumps(tree, indent=2)) > CODEX_INPUT_CAP # the old shape overflowed
+
+ result = _generator().build_overview_structure(tree, [], str(tmp_path))
+ assert len(json.dumps(result, indent=2)) < CODEX_INPUT_CAP / 10