diff --git a/stdlib/tests/stdlib_tests.rs b/stdlib/tests/stdlib_tests.rs index 05c13a8c..51fdb1dc 100644 --- a/stdlib/tests/stdlib_tests.rs +++ b/stdlib/tests/stdlib_tests.rs @@ -519,6 +519,7 @@ fn test_http_module() { let response = "HTTP/1.1 200 OK\r\nContent-Length: 13\r\nContent-Type: text/plain\r\nConnection: close\r\n\r\nHello Server!"; let _ = stream.write_all(response.as_bytes()); let _ = stream.flush(); + let _ = stream.shutdown(std::net::Shutdown::Both); } }); diff --git a/tools/scripts/check_graphify.py b/tools/scripts/check_graphify.py new file mode 100644 index 00000000..4c489e29 --- /dev/null +++ b/tools/scripts/check_graphify.py @@ -0,0 +1,311 @@ +# tools/check_graphify.py +import os +import sys +import json +import re +import subprocess +from pathlib import Path + +# Standardized Exit Codes +EXIT_SUCCESS = 0 +EXIT_VALIDATION_FAILED = 1 +EXIT_MISSING_DEPENDENCY = 2 +EXIT_EXECUTION_FAILED = 3 +EXIT_INVALID_OUTPUT = 4 +EXIT_CONFIG_ERROR = 5 + +def find_repo_root(): + current = Path(__file__).resolve().parent + for _ in range(10): + if (current / ".git").exists() or (current / "Cargo.toml").exists() or (current / ".graphifyignore").exists(): + return current + parent = current.parent + if parent == current: + break + current = parent + return Path(__file__).resolve().parent.parent + +def check_python_dependencies(): + missing = [] + try: + import dotenv + except ImportError: + missing.append("python-dotenv") + try: + import requests + except ImportError: + missing.append("requests") + if sys.version_info < (3, 11): + try: + import tomli + except ImportError: + missing.append("tomli") + return missing + +def validate_graphify_outputs(output_dir): + """ + Dynamically scans output_dir and validates generated files. + Excludes files starting with '.' and manifest.json or cache folder. + Returns (success, results) + """ + if not output_dir.exists(): + return False, {"error": f"Output directory does not exist: {output_dir}"} + + # Get all files, excluding .*, manifest.json, cache/ + generated_files = [] + try: + for p in output_dir.iterdir(): + if p.name.startswith("."): + continue + if p.name == "manifest.json": + continue + if p.is_dir() and p.name == "cache": + continue + if p.is_file(): + generated_files.append(p) + except Exception as e: + return False, {"error": f"Failed to list output directory: {e}"} + + if not generated_files: + return False, {"error": f"No generated output files found in {output_dir}"} + + validation_results = {} + overall_success = True + + for f in generated_files: + filename = f.name + size = f.stat().st_size + if size == 0: + validation_results[filename] = {"status": "FAIL", "reason": "File is empty (0 bytes)"} + overall_success = False + continue + + if f.suffix == ".json": + try: + with open(f, "r", encoding="utf-8") as file: + data = json.load(file) + if not isinstance(data, dict): + validation_results[filename] = {"status": "FAIL", "reason": "Root element is not a JSON object"} + overall_success = False + elif "nodes" not in data or ("edges" not in data and "links" not in data): + validation_results[filename] = {"status": "FAIL", "reason": "Missing 'nodes' and either 'edges' or 'links' keys"} + overall_success = False + else: + edges_key = "links" if "links" in data else "edges" + validation_results[filename] = { + "status": "PASS", + "info": f"Valid JSON with {len(data.get('nodes', []))} nodes and {len(data.get(edges_key, []))} {edges_key} ({size / 1024:.2f} KB)" + } + except json.JSONDecodeError as e: + validation_results[filename] = {"status": "FAIL", "reason": f"Invalid JSON format: {e}"} + overall_success = False + except Exception as e: + validation_results[filename] = {"status": "FAIL", "reason": f"Error reading file: {e}"} + overall_success = False + + elif f.suffix == ".html": + try: + with open(f, "r", encoding="utf-8") as file: + content = file.read(500) # Read start + if "" not in content.lower() and "" not in content.lower(): + validation_results[filename] = {"status": "FAIL", "reason": "Not a valid HTML file"} + overall_success = False + else: + validation_results[filename] = { + "status": "PASS", + "info": f"Valid HTML structure ({size / 1024:.2f} KB)" + } + except Exception as e: + validation_results[filename] = {"status": "FAIL", "reason": f"Error reading file: {e}"} + overall_success = False + + elif f.suffix == ".md": + try: + with open(f, "r", encoding="utf-8") as file: + content = file.read(500) + # Check for markdown headers or content + if not any(line.strip().startswith("#") for line in content.splitlines()): + validation_results[filename] = {"status": "FAIL", "reason": "Markdown does not contain headings"} + overall_success = False + else: + validation_results[filename] = { + "status": "PASS", + "info": f"Valid Markdown with headings ({size / 1024:.2f} KB)" + } + except Exception as e: + validation_results[filename] = {"status": "FAIL", "reason": f"Error reading file: {e}"} + overall_success = False + else: + # Other file types, check size only + validation_results[filename] = { + "status": "PASS", + "info": f"Generic file checked ({size / 1024:.2f} KB)" + } + + return overall_success, validation_results + +def main(): + print("==================================================") + print(" Graphify Verification Utility ") + print("==================================================") + + workspace_dir = find_repo_root() + output_dir = workspace_dir / "graphify-out" + + checks = {} + + # 1. Python version check + py_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + if sys.version_info >= (3, 10): + checks["Python Version"] = ("PASS", f"Python {py_version} (>= 3.10)") + else: + checks["Python Version"] = ("FAIL", f"Python {py_version} is unsupported (requires >= 3.10)") + + # 2. Python dependencies check + missing_deps = check_python_dependencies() + if not missing_deps: + checks["Python Dependencies"] = ("PASS", "All packages imported successfully") + else: + checks["Python Dependencies"] = ("FAIL", f"Missing packages: {', '.join(missing_deps)}. Run: pip install -r requirements.txt") + + # 3. Rust toolchain check + rustc_ver = None + try: + res = subprocess.run(["rustc", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + rustc_ver = res.stdout.strip() + except Exception: + pass + + if rustc_ver: + checks["Rust Installed"] = ("PASS", rustc_ver) + else: + checks["Rust Installed"] = ("FAIL", "rustc is not installed. Install via https://rustup.rs/") + + # 4. Cargo check + cargo_ver = None + try: + res = subprocess.run(["cargo", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + cargo_ver = res.stdout.strip() + except Exception: + pass + + if cargo_ver: + checks["Cargo Installed"] = ("PASS", cargo_ver) + else: + checks["Cargo Installed"] = ("FAIL", "cargo is not installed. Install via https://rustup.rs/") + + # 5. Graphify CLI check + graphify_cli_ver = None + try: + res = subprocess.run(["graphify", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + graphify_cli_ver = res.stdout.strip() + except Exception: + pass + + if not graphify_cli_ver: + # Try via python -m graphify + try: + res = subprocess.run([sys.executable, "-m", "graphify", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + graphify_cli_ver = f"python -m {res.stdout.strip()}" + except Exception: + pass + + if graphify_cli_ver: + match = re.search(r"(\d+\.\d+\.\d+)", graphify_cli_ver) + if match: + checks["Graphify CLI"] = ("PASS", f"Graphify version {match.group(1)}") + else: + checks["Graphify CLI"] = ("PASS", f"Graphify version found but unparsed format: {graphify_cli_ver}") + else: + checks["Graphify CLI"] = ("FAIL", "Graphify is not installed. Run: pip install -r requirements.txt") + + # 6. Configuration Check (.graphifyignore) + ignore_path = workspace_dir / ".graphifyignore" + if ignore_path.exists(): + try: + with open(ignore_path, "r", encoding="utf-8") as f: + content = f.read() + if "graphify-out/" in content or "graphify-out" in content: + checks["Graphify Config"] = ("PASS", ".graphifyignore exists and contains graphify-out") + else: + checks["Graphify Config"] = ("FAIL", ".graphifyignore exists but does not ignore graphify-out/ to prevent recursion") + except Exception as e: + checks["Graphify Config"] = ("FAIL", f"Failed to read .graphifyignore: {e}") + else: + checks["Graphify Config"] = ("FAIL", ".graphifyignore is missing") + + # 7. Output directory check + if output_dir.exists() and output_dir.is_dir(): + checks["Output Directory"] = ("PASS", f"Exists: {output_dir}") + else: + checks["Output Directory"] = ("FAIL", f"Does not exist: {output_dir}") + + # Print environment checklist + print("\nEnvironment & Configuration Checks:") + print("-----------------------------------") + env_failed = False + for name, (status, detail) in checks.items(): + symbol = "[OK]" if status == "PASS" else "[FAIL]" + print(f"[{status}] {symbol} {name}: {detail}") + if status == "FAIL": + env_failed = True + + # Run dynamic output validation if output dir exists + output_failed = False + output_results = {} + if checks["Output Directory"][0] == "PASS": + print("\nDynamic Output Validation:") + print("---------------------------") + valid, results = validate_graphify_outputs(output_dir) + output_results = results + if not valid: + output_failed = True + if "error" in results: + print(f"[FAIL] [ERROR] Error scanning: {results['error']}") + else: + for file, res in results.items(): + symbol = "[OK]" if res["status"] == "PASS" else "[FAIL]" + reason = f" ({res['reason']})" if res["status"] == "FAIL" else f" - {res.get('info', '')}" + print(f"[{res['status']}] {symbol} {file}{reason}") + else: + for file, res in results.items(): + print(f"[PASS] [OK] {file} - {res.get('info', '')}") + else: + output_failed = True + + print("\n==================================================") + + # Determine exit code and print summary + if env_failed: + if checks["Python Version"][0] == "FAIL": + print("Summary: FAIL (Python version unsupported)") + sys.exit(EXIT_VALIDATION_FAILED) + elif checks["Python Dependencies"][0] == "FAIL": + print("Summary: FAIL (Missing Python dependencies)") + sys.exit(EXIT_MISSING_DEPENDENCY) + elif checks["Graphify CLI"][0] == "FAIL": + print("Summary: FAIL (Graphify CLI not installed)") + sys.exit(EXIT_MISSING_DEPENDENCY) + elif checks["Graphify Config"][0] == "FAIL": + print("Summary: FAIL (Graphify configuration error)") + sys.exit(EXIT_CONFIG_ERROR) + elif checks["Rust Installed"][0] == "FAIL" or checks["Cargo Installed"][0] == "FAIL": + print("Summary: FAIL (Rust/Cargo toolchain missing)") + sys.exit(EXIT_MISSING_DEPENDENCY) + else: + print("Summary: FAIL (Environment checks failed)") + sys.exit(EXIT_VALIDATION_FAILED) + + if output_failed: + print("Summary: FAIL (Graph generation outputs missing or invalid)") + sys.exit(EXIT_INVALID_OUTPUT) + + print("Summary: PASS (All checks completed successfully)") + sys.exit(EXIT_SUCCESS) + +if __name__ == "__main__": + main() diff --git a/tools/scripts/update_graphify.py b/tools/scripts/update_graphify.py new file mode 100644 index 00000000..f4e9f955 --- /dev/null +++ b/tools/scripts/update_graphify.py @@ -0,0 +1,323 @@ +# tools/update_graphify.py +import os +import sys +import subprocess +import time +import re +from pathlib import Path + +# Add tools directory to sys.path to import check_graphify +tools_dir = Path(__file__).resolve().parent +if str(tools_dir) not in sys.path: + sys.path.insert(0, str(tools_dir)) + +# Standardized Exit Codes +EXIT_SUCCESS = 0 +EXIT_VALIDATION_FAILED = 1 +EXIT_MISSING_DEPENDENCY = 2 +EXIT_EXECUTION_FAILED = 3 +EXIT_INVALID_OUTPUT = 4 +EXIT_CONFIG_ERROR = 5 + +def check_python_version(): + py_version = f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}" + if sys.version_info < (3, 10): + print(f"[ERROR] Python {py_version} is unsupported (requires >= 3.10)") + sys.exit(EXIT_VALIDATION_FAILED) + return py_version + +# Verify Python dependencies before importing them (self-healing / friendly messages) +def check_python_dependencies(): + missing = [] + try: + import dotenv + except ImportError: + missing.append("python-dotenv") + try: + import requests + except ImportError: + missing.append("requests") + if sys.version_info < (3, 11): + try: + import tomli + except ImportError: + missing.append("tomli") + + if missing: + print(f"[ERROR] Missing Python package(s): {', '.join(missing)}") + print("Fix:") + print(" pip install -r requirements.txt") + sys.exit(EXIT_MISSING_DEPENDENCY) + +# Verify dependencies and import +check_python_version() +check_python_dependencies() + +# Now we can safely import dotenv +import dotenv + +def find_repo_root(): + current = Path(__file__).resolve().parent + for _ in range(10): + if (current / ".git").exists() or (current / "Cargo.toml").exists() or (current / ".graphifyignore").exists(): + return current + parent = current.parent + if parent == current: + break + current = parent + return Path(__file__).resolve().parent.parent + +# Set up Workspace Paths +WORKSPACE_DIR = find_repo_root() +DOTENV_PATH = WORKSPACE_DIR / ".env" +GRAPHIFY_IGNORE_PATH = WORKSPACE_DIR / ".graphifyignore" +OUTPUT_DIR = WORKSPACE_DIR / "graphify-out" +LOG_DIR = WORKSPACE_DIR / "logs" +LOG_FILE = LOG_DIR / "graphify.log" + +def log_message(msg, level="INFO"): + formatted = f"[{time.strftime('%Y-%m-%d %H:%M:%S')}] [{level}] {msg}" + print(formatted) + try: + os.makedirs(LOG_DIR, exist_ok=True) + with open(LOG_FILE, "a", encoding="utf-8") as f: + f.write(formatted + "\n") + except Exception as e: + print(f"[WARNING] Could not write to log file: {e}") + +def check_graphify_installed(): + # Check if CLI is available directly + try: + res = subprocess.run(["graphify", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + version_str = res.stdout.strip() + match = re.search(r"(\d+\.\d+\.\d+)", version_str) + if match: + return True, ["graphify"], match.group(1) + return True, ["graphify"], version_str + except Exception: + pass + + # Check python module path + python_exe = sys.executable + try: + res = subprocess.run([python_exe, "-m", "graphify", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + version_str = res.stdout.strip() + match = re.search(r"(\d+\.\d+\.\d+)", version_str) + if match: + return True, [python_exe, "-m", "graphify"], match.group(1) + return True, [python_exe, "-m", "graphify"], version_str + except Exception: + pass + + log_message("Graphify CLI is not installed.", "ERROR") + print("Fix:") + print(" pip install -r requirements.txt") + sys.exit(EXIT_MISSING_DEPENDENCY) + +def check_rust_toolchain(): + rustc_version = None + cargo_version = None + try: + res = subprocess.run(["rustc", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + rustc_version = res.stdout.strip() + except Exception: + pass + try: + res = subprocess.run(["cargo", "--version"], capture_output=True, text=True, check=False) + if res.returncode == 0: + cargo_version = res.stdout.strip() + except Exception: + pass + return rustc_version, cargo_version + +def check_api_keys(): + keys = ["GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "ANTHROPIC_API_KEY", "DEEPSEEK_API_KEY"] + for key in keys: + if os.environ.get(key): + return key + + if DOTENV_PATH.exists(): + try: + # Load environment variables from .env + dotenv.load_dotenv(DOTENV_PATH) + for key in keys: + if os.environ.get(key): + return key + except Exception as e: + log_message(f"Failed to read .env file: {e}", "WARNING") + return None + +def run_command(cmd_args, dry_run=False): + cmd_str = " ".join(cmd_args) + log_message(f"Executing: {cmd_str}") + if dry_run: + log_message(f"[DRY-RUN] Would run command: {cmd_str}") + return True + + start_time = time.time() + result = subprocess.run(cmd_args, cwd=WORKSPACE_DIR, capture_output=True, text=True) + duration = time.time() - start_time + + if result.returncode != 0: + log_message(f"Command failed in {duration:.2f}s with exit code {result.returncode}", "ERROR") + if result.stderr.strip(): + log_message(f"Error output:\n{result.stderr.strip()}", "ERROR") + return False + + log_message(f"Command completed in {duration:.2f}s") + return True + +def verify_environment(): + """Verify repository structure, Python, toolchains, and LLM keys.""" + if not GRAPHIFY_IGNORE_PATH.exists(): + log_message(f"Configuration error: .graphifyignore is missing at {GRAPHIFY_IGNORE_PATH}", "ERROR") + sys.exit(EXIT_CONFIG_ERROR) + + log_message(f"Workspace root detected: {WORKSPACE_DIR}") + + py_ver = check_python_version() + log_message(f"Python Version: {py_ver}") + + installed, base_cmd, graphify_ver = check_graphify_installed() + log_message(f"Graphify Version: {graphify_ver} (command: {' '.join(base_cmd)})") + + rustc_ver, cargo_ver = check_rust_toolchain() + if rustc_ver: + log_message(f"Rust Version: {rustc_ver}") + else: + log_message("rustc is not installed.", "WARNING") + + if cargo_ver: + log_message(f"Cargo Version: {cargo_ver}") + else: + log_message("cargo is not installed.", "WARNING") + + use_cargo = cargo_ver is not None + if not use_cargo: + log_message("Proceeding without --cargo flag due to missing Cargo installation.", "WARNING") + + active_key = check_api_keys() + if active_key: + log_message(f"Found active API key: {active_key}") + else: + log_message("No LLM API key detected. Running in AST-only mode.", "INFO") + + return base_cmd, use_cargo, active_key + +def run_extraction(base_cmd, use_cargo, active_key, dry_run=False): + """Run Graphify extraction phase and clustering.""" + if not dry_run: + os.makedirs(OUTPUT_DIR, exist_ok=True) + + success = False + if active_key: + cmd = base_cmd + ["extract", "."] + if use_cargo: + cmd.append("--cargo") + success = run_command(cmd, dry_run=dry_run) + else: + log_message("Temporarily modifying .graphifyignore for AST-only fallback...") + + original_ignore = "" + try: + with open(GRAPHIFY_IGNORE_PATH, "r", encoding="utf-8-sig") as f: + original_ignore = f.read() + except Exception as e: + log_message(f"Failed to read .graphifyignore: {e}", "ERROR") + sys.exit(EXIT_CONFIG_ERROR) + + temp_ignore = ( + original_ignore + + "\ndocs/\n.agents/\n.github/\nresearch/\nassets/\nreleases/\nlogs/\nscripts/\neditors/\npython-installer/\npython-installer-lang/\ndesign/\n" + "Executive Summary.pdf\n*.md\n**/*.md\n*.pdf\n**/*.pdf\n" + "*.docx\n**/*.docx\n*.txt\n**/*.txt\n*.png\n**/*.png\n" + "*.jpg\n**/*.jpg\n*.jpeg\n**/*.jpeg\n*.gif\n**/*.gif\n" + "*.svg\n**/*.svg\n*.ico\n**/*.ico\n*.sh\n**/*.sh\n" + "requirements.txt\n*.bat\n**/*.bat\n*.toml\n**/*.toml\n" + "*.yml\n**/*.yml\n*.yaml\n**/*.yaml\nmkdocs.yml\n" + "*.lock\n**/*.lock\n" + "LICENSE\n**/LICENSE\n" + "NOTICE\n**/NOTICE\n" + "COPYRIGHT\n**/COPYRIGHT\n" + ".graphifyignore\n" + ) + + ignore_modified = False + try: + if not dry_run: + with open(GRAPHIFY_IGNORE_PATH, "w", encoding="utf-8") as f: + f.write(temp_ignore) + ignore_modified = True + log_message(f"Wrote temporary ignore file to {GRAPHIFY_IGNORE_PATH}") + + cmd = base_cmd + ["extract", ".", "--no-cluster", "--code-only"] + if use_cargo: + cmd.append("--cargo") + success = run_command(cmd, dry_run=dry_run) + finally: + if ignore_modified: + try: + with open(GRAPHIFY_IGNORE_PATH, "w", encoding="utf-8") as f: + f.write(original_ignore) + log_message("Restored original .graphifyignore") + except Exception as e: + log_message(f"Failed to restore .graphifyignore: {e}", "ERROR") + sys.exit(EXIT_CONFIG_ERROR) + + if not success: + log_message("Graphify extraction failed.", "ERROR") + sys.exit(EXIT_EXECUTION_FAILED) + + log_message("Running clustering and generating reports...") + cmd = base_cmd + ["cluster-only", "."] + if not active_key: + cmd.append("--no-label") + if not run_command(cmd, dry_run=dry_run): + log_message("Clustering failed.", "ERROR") + sys.exit(EXIT_EXECUTION_FAILED) + +def validate_outputs(dry_run=False): + """Validate generated outputs.""" + if dry_run: + return + + try: + from check_graphify import validate_graphify_outputs + valid, results = validate_graphify_outputs(OUTPUT_DIR) + if not valid: + log_message("Output validation failed.", "ERROR") + if "error" in results: + log_message(results["error"], "ERROR") + else: + for f, r in results.items(): + if r["status"] == "FAIL": + log_message(f"{f}: {r['reason']}", "ERROR") + sys.exit(EXIT_INVALID_OUTPUT) + + log_message("Output validation succeeded:") + for f, r in results.items(): + log_message(f" - {f}: {r.get('info', 'Verified')}") + except ImportError: + log_message("Warning: check_graphify.py validation module could not be imported.", "WARNING") + +def main(): + start_time = time.time() + dry_run = "--dry-run" in sys.argv + + log_message("=" * 60) + log_message(f"Starting Graphify Knowledge Graph Update (dry-run: {dry_run})") + + base_cmd, use_cargo, active_key = verify_environment() + run_extraction(base_cmd, use_cargo, active_key, dry_run=dry_run) + validate_outputs(dry_run=dry_run) + + execution_time = time.time() - start_time + log_message(f"Graphify update completed successfully in {execution_time:.2f}s.") + log_message("=" * 60 + "\n") + sys.exit(EXIT_SUCCESS) + +if __name__ == "__main__": + main()