From 83c10dba5d45eba3f52e16a366f74a12b0f5043c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 22 Apr 2026 03:41:41 +0200 Subject: [PATCH 01/50] feat: create pipeline for symbol reachability and add a test Signed-off-by: ziad hany --- pyproject.toml | 1 + .../pipelines/collect_symbols_reachability.py | 35 + scanpipe/pipes/reachability.py | 730 ++++++++++++++++++ scanpipe/pipes/symbols.py | 155 ++++ scanpipe/tests/data/reachability/app.py | 35 + .../tests/data/reachability/diff-app.patch | 39 + scanpipe/tests/data/reachability/fixed-app.py | 41 + scanpipe/tests/data/reachability/vuln-app.py | 35 + .../tests/pipes/test_symbols_reachability.py | 293 +++++++ 9 files changed, 1364 insertions(+) create mode 100644 scanpipe/pipelines/collect_symbols_reachability.py create mode 100644 scanpipe/pipes/reachability.py create mode 100644 scanpipe/tests/data/reachability/app.py create mode 100644 scanpipe/tests/data/reachability/diff-app.patch create mode 100644 scanpipe/tests/data/reachability/fixed-app.py create mode 100644 scanpipe/tests/data/reachability/vuln-app.py create mode 100644 scanpipe/tests/pipes/test_symbols_reachability.py diff --git a/pyproject.toml b/pyproject.toml index 350c59a0b7..4b3f331a1e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,6 +151,7 @@ run = "scancodeio:combined_run" analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker" analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS" analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows" +analyze_symbols_reachability = "scanpipe.pipelines.collect_symbols_reachability:SymbolReachability" benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls" collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext" collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags" diff --git a/scanpipe/pipelines/collect_symbols_reachability.py b/scanpipe/pipelines/collect_symbols_reachability.py new file mode 100644 index 0000000000..15519fc661 --- /dev/null +++ b/scanpipe/pipelines/collect_symbols_reachability.py @@ -0,0 +1,35 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import reachability + + +class SymbolReachability(Pipeline): + """ + Patch reachability analysis, for given a vulnerability patches + """ + + download_inputs = False + is_addon = True + results_url = "/project/{slug}/resources/?extra_data=symbol_reachability" + + @classmethod + def steps(cls): + return (cls.analyze_and_store_symbol_reachability,) + + def analyze_and_store_symbol_reachability(self): + """ + Perform symbol-level reachability analysis for each patch. + This step compares the AST of patched/vulnerable files against the codebase resources. + Results are stored directly in the 'extra_data' of each CodebaseResource. + """ + reachability.collect_and_store_symbol_reachability_results( + project=self.project, logger=self.log + ) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py new file mode 100644 index 0000000000..5531a1f495 --- /dev/null +++ b/scanpipe/pipes/reachability.py @@ -0,0 +1,730 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import os +import shutil +import tempfile +from enum import Enum +from pathlib import Path + +from git import Repo +from git.diff import NULL_TREE +from git.exc import BadName +from matchcode_toolkit.fingerprinting import create_file_fingerprints +from scancode.api import get_file_info +from unidiff import PatchSet + +from scanpipe.pipes.symbols import TS_QUERIES +from scanpipe.pipes.symbols import _root_of +from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.symbols import extract_calls_in_node +from scanpipe.pipes.symbols import extract_definitions +from scanpipe.pipes.symbols import extract_symbols +from scanpipe.pipes.symbols import parse_code_to_ast +from scanpipe.pipes.symbols import qualified_name_from_index + +EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" + + +class ReachabilityStatus(str, Enum): + REACHABLE = "REACHABLE" + POTENTIALLY_REACHABLE = "POTENTIALLY_REACHABLE" + NOT_REACHABLE = "NOT_REACHABLE" + + +def api_mocker(): + """ + TODO: Remove this once the API patch url is done + """ + return [ + { + "vcs_url": "https://github.com/pallets/flask", + "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", + }, + ] + + +def clone_repo(vcs_url, commit_hash=None): + repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") + + try: + repo = Repo.clone_from(vcs_url, repo_path) + + if commit_hash: + repo.git.checkout(commit_hash) + + return repo_path + + except BadName as exc: + cleanup_repo(repo_path) + raise ValueError(f"Commit {commit_hash} not found") from exc + + except Exception: + cleanup_repo(repo_path) + raise + + +def cleanup_repo(repo_path): + if repo_path and os.path.exists(repo_path): + shutil.rmtree(repo_path, ignore_errors=True) + + +def normalize_text(content): + if content is None: + return "" + + if isinstance(content, bytes): + return content.decode("utf-8", errors="replace") + + return str(content) + + +def is_supported_language(language): + """A language is supported if we have tree-sitter queries for it.""" + return bool(language) and language in TS_QUERIES + + +def detect_language_with_scancode(file_path, content): + """ + Write `content` to a temp file preserving `file_path`'s basename + so the extension is meaningful, then ask ScanCode's `get_file_info` + to return the programming language. + """ + content = normalize_text(content) + + if not content: + return None + + tmp_dir = tempfile.mkdtemp(prefix="patch-lang-") + + try: + target = Path(tmp_dir) / Path(file_path).name + target.write_text(content, encoding="utf-8", errors="replace") + + info = get_file_info(location=str(target)) or {} + return info.get("programming_language") or None + + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + +def get_commit_and_parent(repo, commit_hash): + commit = repo.commit(commit_hash) + parent = commit.parents[0] if commit.parents else None + return commit, parent + + +def get_commit_diff_text(repo, parent_commit, commit): + """Whole-commit unified diff (used to extract changed line numbers).""" + base = parent_commit.hexsha if parent_commit else EMPTY_TREE_SHA + return repo.git.diff(base, commit.hexsha, unified=0) + + +def get_changed_files(parent_commit, commit): + """ + Return: + { + file_path: { + "vulnerable_text": "...", + "fixed_text": "...", + } + } + + """ + diffs = ( + parent_commit.diff(commit, create_patch=False) + if parent_commit + else commit.diff(NULL_TREE, create_patch=False) + ) + + files = {} + for diff in diffs: + change_type = diff.change_type + old_path = diff.a_path if change_type in ("D", "M", "R") else None + new_path = diff.b_path if change_type in ("A", "M", "R") else None + path_key = new_path or old_path + + if not path_key: + continue + + entry = files.setdefault( + path_key, + { + "vulnerable_text": "", + "fixed_text": "", + }, + ) + + if old_path and parent_commit: + entry["vulnerable_text"] = ( + (parent_commit.tree / old_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) + + if new_path: + entry["fixed_text"] = ( + (commit.tree / new_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) + + return files + + +def get_changed_lines(diff_text, file_path): + """Return `(removed_lines, added_lines)` for one file from a unified diff.""" + removed = [] + added = [] + + if not diff_text: + return removed, added + + for patched_file in PatchSet.from_string(diff_text): + candidates = { + patched_file.path, + (patched_file.source_file or "").removeprefix("a/"), + (patched_file.target_file or "").removeprefix("b/"), + } + + if file_path not in candidates: + continue + + for hunk in patched_file: + for line in hunk: + if line.is_removed and line.source_line_no: + removed.append(line.source_line_no) + elif line.is_added and line.target_line_no: + added.append(line.target_line_no) + + return removed, added + + +def query_captures(language, kind, node): + """ + Re-run a definition query on the root of `node`'s tree so ancestors can + be compared. Query caching is handled by `scanpipe.pipes.symbols`. + """ + from scanpipe.pipes.symbols import get_query + from scanpipe.pipes.symbols import run_query + + root = node + + while root.parent is not None: + root = root.parent + + query = get_query(language, kind) + return list(run_query(query, root)) + + +def is_nested_function(node, language): + function_nodes = { + captured_node + for captured_node, _ in query_captures(language, "functions", node) + } + class_nodes = { + captured_node for captured_node, _ in query_captures(language, "classes", node) + } + + if node not in function_nodes: + return False + + function_types = {captured_node.type for captured_node in function_nodes} + class_types = {captured_node.type for captured_node in class_nodes} + + parent = node.parent + + while parent is not None: + if parent.type in function_types: + return True + + if parent.type in class_types: + return False + + parent = parent.parent + + return False + + +def diff_changed_symbols(vuln_meta, fixed_meta): + """ + Keep only symbols whose body actually differs between vulnerable and fixed + versions. Pair by qualified name first. + """ + fixed_by_qn = { + metadata["qualified_name"]: metadata for metadata in fixed_meta.values() + } + + vuln_by_qn = { + metadata["qualified_name"]: metadata for metadata in vuln_meta.values() + } + + vuln_only = { + key: metadata + for key, metadata in vuln_meta.items() + if fixed_by_qn.get(metadata["qualified_name"], {}).get("text") + != metadata["text"] + } + + fixed_only = { + key: metadata + for key, metadata in fixed_meta.items() + if vuln_by_qn.get(metadata["qualified_name"], {}).get("text") + != metadata["text"] + } + + return vuln_only, fixed_only + + +def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): + """ + Return `(vuln_metadata, fixed_metadata, language)` for one changed file, + restricted to symbols actually touched by the patch. + """ + vulnerable_text = normalize_text(vulnerable_text) + fixed_text = normalize_text(fixed_text) + + language = detect_language_with_scancode( + file_path, fixed_text + ) or detect_language_with_scancode(file_path, vulnerable_text) + + if not is_supported_language(language): + return {}, {}, language + + vuln_tree, _ = ( + parse_code_to_ast(vulnerable_text, language) + if vulnerable_text + else (None, None) + ) + + fixed_tree, _ = ( + parse_code_to_ast(fixed_text, language) if fixed_text else (None, None) + ) + + if vuln_tree is None and fixed_tree is None: + return {}, {}, language + + removed_lines, added_lines = get_changed_lines(diff_text, file_path) + + vuln_nodes = ( + extract_symbols(vuln_tree, removed_lines, language) if vuln_tree else [] + ) + + fixed_nodes = ( + extract_symbols(fixed_tree, added_lines, language) if fixed_tree else [] + ) + + vuln_meta, fixed_meta = diff_changed_symbols( + build_symbol_metadata(vuln_nodes, language), + build_symbol_metadata(fixed_nodes, language), + ) + + return vuln_meta, fixed_meta, language + + +def collect_patch_symbols(repo, commit_hash): + """ + Return: + { + language: { + "vulnerable": { + "file_path::symbol_key": metadata, + ... + }, + "fixed": { + "file_path::symbol_key": metadata, + ... + }, + }, + ... + } + + Symbols are bucketed by language so resources are only matched against + patch symbols extracted from the same language. + + """ + commit, parent = get_commit_and_parent(repo, commit_hash) + diff_text = get_commit_diff_text(repo, parent, commit) + changed = get_changed_files(parent, commit) + + by_language = {} + for file_path, texts in changed.items(): + vulnerable_text = texts["vulnerable_text"] + fixed_text = texts["fixed_text"] + vuln_meta, fixed_meta, language = analyze_patched_file( + vulnerable_text=vulnerable_text, + fixed_text=fixed_text, + diff_text=diff_text, + file_path=file_path, + ) + + if not language or not (vuln_meta or fixed_meta): + continue + + language_bucket = by_language.setdefault( + language, + { + "vulnerable": {}, + "fixed": {}, + }, + ) + + language_bucket["vulnerable"].update( + {f"{file_path}::{key}": metadata for key, metadata in vuln_meta.items()} + ) + + language_bucket["fixed"].update( + {f"{file_path}::{key}": metadata for key, metadata in fixed_meta.items()} + ) + + return by_language + + +def append_symbol_reachability_result(resource, result): + """ + Append one symbol reachability result to the resource extra_data without + overwriting previous results. + """ + extra_data = resource.extra_data or {} + existing_results = extra_data.get("symbols_reachability", []) + + if not isinstance(existing_results, list): + existing_results = [existing_results] + + existing_results.append(result) + + resource.update_extra_data( + { + "symbols_reachability": existing_results, + } + ) + + +def collect_and_store_symbol_reachability_results(project, logger=None): + """ + For each known patch commit, determine whether each project codebase + resource is reachable to the vulnerable code by comparing tree-sitter ASTs + of the patch versus the resource. + + Result classification: + - REACHABLE + - POTENTIALLY_REACHABLE + - NOT_REACHABLE + """ + candidate_resources = project.codebaseresources.files().filter( + is_binary=False, + is_archive=False, + is_media=False, + ) + + for patch in api_mocker(): + vcs_url = patch["vcs_url"] + commit_hash = patch["commit_hash"] + repo_path = None + try: + repo_path = clone_repo(vcs_url, commit_hash) + repo = Repo(repo_path) + + patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) + + if not patch_symbols_by_language: + continue + + for resource in candidate_resources: + resource_language = resource.programming_language + + if resource_language not in patch_symbols_by_language: + continue + + resource_text = normalize_text(resource.file_content) + + if not resource_text: + continue + + patch_symbols = patch_symbols_by_language[resource_language] + vuln_metadata = patch_symbols["vulnerable"] + fixed_metadata = patch_symbols["fixed"] + + resource_index = build_resource_index( + resource_text, + resource_language, + ) + + if not resource_index: + continue + + vuln_match_symbols = match_symbols_against_resource( + vuln_metadata, + resource_index, + ) + + fixed_match_symbols = match_symbols_against_resource( + fixed_metadata, + resource_index, + ) + + if not vuln_match_symbols and not fixed_match_symbols: + continue + + result = { + "reachability_status": classify_reachability(vuln_match_symbols), + "summary": { + "vulnerable_symbols": sorted(vuln_match_symbols), + "fixed_symbols": sorted(fixed_match_symbols), + "call_paths": { + qn: ev.get("reachable_from", []) + for qn, ev in vuln_match_symbols.items() + if ev.get("called") + }, + }, + "evidence": vuln_match_symbols, + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + } + + append_symbol_reachability_result(resource, result) + + except Exception as e: + logger.exception( + "Failed to collect symbol reachability for " + f"{vcs_url}@{commit_hash}: {e}" + ) + finally: + cleanup_repo(repo_path) + + +def compute_reachable_symbols(call_graph, target_simple_names): + if not call_graph or not target_simple_names: + return set(), False + + edges = call_graph["edges"] + targets = set(target_simple_names) + + callers_of = {} + for caller_qn, callees in edges.items(): + for callee_simple in callees: + callers_of.setdefault(callee_simple, set()).add(caller_qn) + + direct_callers = set() + for target in targets: + direct_callers |= callers_of.get(target, set()) + + has_direct_call = bool(direct_callers) + + by_simple = call_graph["by_simple_name"] + qn_to_simple = {qn: meta["simple_name"] for qn, meta in call_graph["nodes"].items()} + + reachable = set(direct_callers) + frontier = list(direct_callers) + + while frontier: + current_qn = frontier.pop() + current_simple = qn_to_simple.get(current_qn) + if not current_simple: + continue + for parent_qn in callers_of.get(current_simple, ()): + if parent_qn not in reachable: + reachable.add(parent_qn) + frontier.append(parent_qn) + + return reachable, has_direct_call + + +def build_resource_index(resource_text, language): + resource_text = normalize_text(resource_text) + + if not is_supported_language(language) or not resource_text: + return None + + tree, _ = parse_code_to_ast(resource_text, language) + + if tree is None: + return None + + call_graph = build_call_graph(tree, language) + + meta = ( + call_graph["nodes"] + if call_graph + else build_symbol_metadata( + extract_definitions(tree, language), + language, + ) + ) + + return { + "definitions": {metadata["qualified_name"] for metadata in meta.values()}, + "fingerprints": { + metadata["fingerprint"] + for metadata in meta.values() + if metadata["fingerprint"] + }, + "call_graph": call_graph, + } + + +def match_symbols_against_resource(symbols, resource_index): + if not symbols or not resource_index: + return {} + + call_graph = resource_index.get("call_graph") + target_simple_names = {metadata["simple_name"] for metadata in symbols.values()} + + reachable_callers, _has_direct = compute_reachable_symbols( + call_graph, + target_simple_names, + ) + + called_simple_names = set() + if call_graph: + for callees in call_graph["edges"].values(): + called_simple_names |= callees + + matched = {} + for metadata in symbols.values(): + qualified_name = metadata["qualified_name"] + simple_name = metadata["simple_name"] + fingerprint = metadata["fingerprint"] + + defined = qualified_name in resource_index["definitions"] + fingerprint_hit = bool( + fingerprint and fingerprint in resource_index["fingerprints"] + ) + called = simple_name in called_simple_names + + if not (defined or fingerprint_hit or called): + continue + + entry = matched.setdefault( + qualified_name, + { + "defined": False, + "called": False, + "reachable_from": [], + }, + ) + + entry["defined"] = entry["defined"] or defined + entry["called"] = entry["called"] or called + + if fingerprint_hit: + entry["exact_match_fingerprint"] = fingerprint + + if called: + entry["reachable_from"] = sorted(reachable_callers) + + return matched + + +def classify_reachability(evidence): + if not evidence: + return ReachabilityStatus.NOT_REACHABLE + + SEVERITY_RANK = { + ReachabilityStatus.NOT_REACHABLE: 0, + ReachabilityStatus.POTENTIALLY_REACHABLE: 1, + ReachabilityStatus.REACHABLE: 2, + } + + highest_status = ReachabilityStatus.NOT_REACHABLE + + for item in evidence.values(): + is_called = bool(item.get("called")) + has_path = bool(item.get("reachable_from")) + is_exact = "exact_match_fingerprint" in item + is_defined = bool(item.get("defined")) + + if is_called or (has_path and is_exact): + return ReachabilityStatus.REACHABLE + + elif has_path or is_exact or is_defined: + current_item_status = ReachabilityStatus.POTENTIALLY_REACHABLE + + else: + current_item_status = ReachabilityStatus.NOT_REACHABLE + + if SEVERITY_RANK[current_item_status] > SEVERITY_RANK[highest_status]: + highest_status = current_item_status + return highest_status + + +def build_symbol_metadata(nodes, language, index=None): + if index is None and nodes: + index = collect_definitions(_root_of(nodes[0]), language) + + metadata = {} + for node in nodes: + if is_nested_function(node, language): + continue + + qualified_name = qualified_name_from_index(node, index) + if not qualified_name: + continue + + body_text = node.text.decode("utf-8", errors="replace") + fingerprints = create_file_fingerprints(content=body_text) or {} + + key = qualified_name + suffix = 1 + while key in metadata: + suffix += 1 + key = f"{qualified_name}#{suffix}" + + metadata[key] = { + "qualified_name": qualified_name, + "simple_name": qualified_name.rsplit(".", 1)[-1], + "text": body_text, + "fingerprint": fingerprints.get("halo1"), + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "node_type": node.type, + } + return metadata + + +def build_call_graph(tree, language): + if tree is None or not is_supported_language(language): + return None + + index = collect_definitions(tree.root_node, language) + definition_nodes = [d["node"] for d in index.values()] + metadata = build_symbol_metadata(definition_nodes, language, index=index) + + qualified_name_to_node = {} + for node in definition_nodes: + qualified_name = qualified_name_from_index(node, index) + if qualified_name: + qualified_name_to_node.setdefault(qualified_name, node) + + edges = {} + by_simple_name = {} + for qualified_name, meta in metadata.items(): + canonical = meta["qualified_name"] + node = qualified_name_to_node.get(canonical) + if node is None: + continue + edges.setdefault(canonical, set()).update(extract_calls_in_node(node, language)) + by_simple_name.setdefault(meta["simple_name"], set()).add(canonical) + + return {"nodes": metadata, "edges": edges, "by_simple_name": by_simple_name} diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 76493d8dac..c6247dbc44 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -20,8 +20,20 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import importlib +from functools import cache + from django.db.models import Q +from source_inspector import symbols_ctags +from source_inspector import symbols_pygments +from source_inspector import symbols_tree_sitter +from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS +from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled +from tree_sitter import Language +from tree_sitter import Parser +from tree_sitter import Query + from aboutcode.pipeline import LoopProgress @@ -171,3 +183,146 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "source_strings": result.get("source_strings"), } ) + + +SYMBOLS_TYPE_SUPPORTED = { + "ctags": symbols_ctags.get_symbols, + "tree_sitter": symbols_tree_sitter.get_treesitter_symbols, + "pygments": symbols_pygments.get_pygments_symbols, +} + +# https://github.com/Aider-AI/aider/tree/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/queries/tree-sitter-language-pack +TS_QUERIES = { + "Python": { + "functions": """ + (function_definition name: (identifier) @name) @function + """, + "classes": """ + (class_definition name: (identifier) @name) @class + """, + "calls": """ + (call function: (identifier) @callee) + (call function: (attribute attribute: (identifier) @callee)) + """, + }, +} + +@cache +def load_language(language: str) -> Language: + if language not in TS_LANGUAGE_WHEELS: + raise ValueError(f"Unsupported language: {language}") + + wheel = TS_LANGUAGE_WHEELS[language]["wheel"] + try: + grammar = importlib.import_module(wheel) + except ModuleNotFoundError as exc: + raise TreeSitterWheelNotInstalled( + f"Grammar wheel '{wheel}' is not installed." + ) from exc + return Language(grammar.language()) + + +@cache +def get_query(language: str, kind: str) -> Query | None: + source = TS_QUERIES.get(language, {}).get(kind, "").strip() + if not source: + return None + return Query(load_language(language), source) + + +def parse_code_to_ast(code_text: str, language: str): + if not code_text or not language or language not in TS_LANGUAGE_WHEELS: + return None, None + + ts_language = load_language(language) + parser = Parser(language=ts_language) + return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[language] + + +def run_query(query: Query, root_node): + """Yield ``(definition_node, name)`` pairs for function/class queries.""" + if query is None: + return + + for _pattern_index, captures in query.matches(root_node): + def_nodes = captures.get("function") or captures.get("class") or [] + if not def_nodes: + continue + + name_nodes = captures.get("name") or [] + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") if name_nodes else None + ) + yield def_nodes[0], name + + +def extract_calls_in_node(node, language: str) -> set[str]: + query = get_query(language, "calls") + if query is None or node is None: + return set() + + names = set() + for _pattern_index, captures in query.matches(node): + for callee_node in captures.get("callee", []): + name = callee_node.text.decode("utf-8", errors="replace") + if name: + names.add(name) + return names + + +def collect_definitions(root_node, language: str) -> dict[int, dict]: + index: dict[int, dict] = {} + for kind in ("functions", "classes"): + query = get_query(language, kind) + for node, name in run_query(query, root_node): + index[node.id] = {"node": node, "name": name, "kind": kind} + return index + + +def extract_definitions(tree, language: str, kinds=("functions", "classes")): + if tree is None: + return [] + index = collect_definitions(tree.root_node, language) + return [d["node"] for d in index.values() if d["kind"] in kinds] + + +def extract_symbols(tree, changed_lines: list[int], language: str): + if tree is None or not changed_lines: + return [] + + definition_ids = set(collect_definitions(tree.root_node, language).keys()) + if not definition_ids: + return [] + + seen = set() + enclosing = [] + + for line in changed_lines: + row = max(0, line - 1) + node = tree.root_node.descendant_for_point_range((row, 0), (row, 0)) + + while node is not None: + if node.id in definition_ids and node.id not in seen: + seen.add(node.id) + enclosing.append(node) + break + node = node.parent + + return enclosing + + +def _root_of(node): + while node.parent is not None: + node = node.parent + return node + + +def qualified_name_from_index(node, index: dict[int, dict]) -> str: + parts = [] + curr = node + while curr is not None: + definition = index.get(curr.id) + if definition is not None and definition["name"]: + parts.append(definition["name"]) + curr = curr.parent + return ".".join(reversed(parts)) diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/app.py new file mode 100644 index 0000000000..c64ae7d9d1 --- /dev/null +++ b/scanpipe/tests/data/reachability/app.py @@ -0,0 +1,35 @@ +import os + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +def serve_report(request_payload): + """Top-level function handling a request.""" + generator = ReportGenerator("/var/reports") + requested_file = request_payload.get("file") + + # Helper function nested inside serve_report + def build_file_path(filename): + # VULNERABLE: Direct concatenation allows Path Traversal + # An attacker passing "../../etc/passwd" could read system files. + return os.path.join(generator.base_dir, filename) + + if not requested_file: + return "Error: No file specified" + + target_path = build_file_path(requested_file) + + if os.path.exists(target_path): + return f"Serving content of {target_path}" + + return "Error: File not found" + + +def unrelated_top_level_function(): + """An extra function to test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/diff-app.patch b/scanpipe/tests/data/reachability/diff-app.patch new file mode 100644 index 0000000000..ccb86953a8 --- /dev/null +++ b/scanpipe/tests/data/reachability/diff-app.patch @@ -0,0 +1,39 @@ +From 8f7b1c3d9a4e2b6f5d8c1a2e3f4b5c6d7e8f9a0b Mon Sep 17 00:00:00 2001 +From: Security Team +Date: Tue, 2 Jun 2026 10:00:00 +0000 +Subject: [PATCH] Fix path traversal vulnerability in report generator + +- Validates that target paths stay within the designated base_dir. +- Catches ValueError on invalid path resolution. +--- + app.py | 12 +++++++++--- + 1 file changed, 9 insertions(+), 3 deletions(-) + +diff --git a/app.py b/app.py +index a1b2c3d..e4f5g6h 100644 +--- a/app.py ++++ b/app.py +@@ -15,13 +15,19 @@ def serve_report(request_payload): + # Helper function nested inside serve_report + def build_file_path(filename): +- # VULNERABLE: Direct concatenation allows Path Traversal +- # An attacker passing "../../etc/passwd" could read system files. +- return os.path.join(generator.base_dir, filename) ++ # FIXED: Validate that the resolved path stays within the base_dir ++ base = os.path.abspath(generator.base_dir) ++ target = os.path.abspath(os.path.join(base, filename)) ++ if not target.startswith(base): ++ raise ValueError("Path Traversal Detected") ++ return target + + if not requested_file: + return "Error: No file specified" + +- target_path = build_file_path(requested_file) ++ try: ++ target_path = build_file_path(requested_file) ++ except ValueError: ++ return "Error: Invalid path" + + if os.path.exists(target_path): + return f"Serving content of {target_path}" \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/fixed-app.py new file mode 100644 index 0000000000..3296bb843e --- /dev/null +++ b/scanpipe/tests/data/reachability/fixed-app.py @@ -0,0 +1,41 @@ +import os + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +def serve_report(request_payload): + """Top-level function handling a request.""" + generator = ReportGenerator("/var/reports") + requested_file = request_payload.get("file") + + # Helper function nested inside serve_report + def build_file_path(filename): + # FIXED: Validate that the resolved path stays within the base_dir + base = os.path.abspath(generator.base_dir) + target = os.path.abspath(os.path.join(base, filename)) + if not target.startswith(base): + raise ValueError("Path Traversal Detected") + return target + + if not requested_file: + return "Error: No file specified" + + try: + target_path = build_file_path(requested_file) + except ValueError: + return "Error: Invalid path" + + if os.path.exists(target_path): + return f"Serving content of {target_path}" + + return "Error: File not found" + + +def unrelated_top_level_function(): + """An extra function to test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/vuln-app.py new file mode 100644 index 0000000000..c64ae7d9d1 --- /dev/null +++ b/scanpipe/tests/data/reachability/vuln-app.py @@ -0,0 +1,35 @@ +import os + + +class ReportGenerator: + """A dummy class to test AST class method parsing.""" + + def __init__(self, base_dir): + self.base_dir = base_dir + + +def serve_report(request_payload): + """Top-level function handling a request.""" + generator = ReportGenerator("/var/reports") + requested_file = request_payload.get("file") + + # Helper function nested inside serve_report + def build_file_path(filename): + # VULNERABLE: Direct concatenation allows Path Traversal + # An attacker passing "../../etc/passwd" could read system files. + return os.path.join(generator.base_dir, filename) + + if not requested_file: + return "Error: No file specified" + + target_path = build_file_path(requested_file) + + if os.path.exists(target_path): + return f"Serving content of {target_path}" + + return "Error: File not found" + + +def unrelated_top_level_function(): + """An extra function to test AST node boundaries.""" + return "I am just here to add AST complexity." diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py new file mode 100644 index 0000000000..30d4c924dd --- /dev/null +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -0,0 +1,293 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +from pathlib import Path +from unittest.mock import patch + +from django.test import TestCase + +from scanpipe.models import Project +from scanpipe.pipes import collect_and_create_codebase_resources +from scanpipe.pipes.reachability import ReachabilityStatus +from scanpipe.pipes.reachability import analyze_patched_file +from scanpipe.pipes.reachability import build_call_graph +from scanpipe.pipes.reachability import classify_reachability +from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.symbols import extract_definitions +from scanpipe.pipes.symbols import parse_code_to_ast +from scanpipe.pipes.symbols import qualified_name_from_index + + +class SymbolReachabilityPipesTest(TestCase): + data = Path(__file__).parent.parent / "data" / "reachability" + + def setUp(self): + self.project1 = Project.objects.create(name="Analysis") + self.project1.codebase_path.mkdir(parents=True, exist_ok=True) + + @patch("scanpipe.pipes.reachability.Repo") + @patch("scanpipe.pipes.reachability.clone_repo") + @patch("scanpipe.pipes.reachability.api_mocker") + @patch("scanpipe.pipes.reachability.collect_patch_symbols") + def test_collect_and_store_symbol_reachability_results( + self, mock_collect_symbols, mock_api, mock_clone_repo, mock_repo + ): + app_text = (self.data / "app.py").read_text() + vuln_text = (self.data / "vuln-app.py").read_text() + fixed_text = (self.data / "fixed-app.py").read_text() + diff_text = (self.data / "diff-app.patch").read_text() + + vuln_meta, fixed_meta, lang = analyze_patched_file( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + diff_text=diff_text, + file_path="app.py", + ) + + self.assertTrue(lang) + self.assertTrue(vuln_meta or fixed_meta) + mock_api.return_value = [ + { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + } + ] + + mock_clone_repo.return_value = str(self.project1.codebase_path) + mock_collect_symbols.return_value = { + lang: { + "vulnerable": { + f"app.py::{key}": metadata for key, metadata in vuln_meta.items() + }, + "fixed": { + f"app.py::{key}": metadata for key, metadata in fixed_meta.items() + }, + } + } + + resource_file = self.project1.codebase_path / "app.py" + resource_file.write_text(app_text) + collect_and_create_codebase_resources(self.project1) + + resource = self.project1.codebaseresources.get(path="app.py") + resource.programming_language = lang + resource.save() + + collect_and_store_symbol_reachability_results(self.project1) + + resource.refresh_from_db() + results = resource.extra_data.get("symbols_reachability") + + assert results == [ + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "summary": { + "call_paths": {}, + "fixed_symbols": ["serve_report"], + "vulnerable_symbols": ["serve_report"], + }, + "evidence": { + "serve_report": { + "called": False, + "defined": True, + "reachable_from": [], + "exact_match_fingerprint": "000000556d322a47595af353274b000aa324e014", + } + }, + "reachability_status": "POTENTIALLY_REACHABLE", + } + ] + + def test_build_call_graph(self): + source_code = """ +def calculate_total(price, tax): + return price + get_tax_amount(price, tax) + +def get_tax_amount(price, tax): + return price * tax + +def process_order(): + total = calculate_total(100, 0.05) + print("Done") +""" + tree, _ = parse_code_to_ast(source_code, "Python") + result = build_call_graph(tree, "Python") + + assert result == { + "nodes": { + "calculate_total": { + "qualified_name": "calculate_total", + "simple_name": "calculate_total", + "text": "def calculate_total(price, tax):\n return price + get_tax_amount(price, tax)", + "fingerprint": "00000008060105fd3624134884412006ce880936", + "start_line": 2, + "end_line": 3, + "node_type": "function_definition", + }, + "get_tax_amount": { + "qualified_name": "get_tax_amount", + "simple_name": "get_tax_amount", + "text": "def get_tax_amount(price, tax):\n return price * tax", + "fingerprint": "000000058f0ee87d9669f20b1f473137b665bb20", + "start_line": 5, + "end_line": 6, + "node_type": "function_definition", + }, + "process_order": { + "qualified_name": "process_order", + "simple_name": "process_order", + "text": 'def process_order():\n total = calculate_total(100, 0.05)\n print("Done")', + "fingerprint": "000000071c3e6902da5c2b322386eff29068e3e2", + "start_line": 8, + "end_line": 10, + "node_type": "function_definition", + }, + }, + "edges": { + "calculate_total": {"get_tax_amount"}, + "get_tax_amount": set(), + "process_order": {"print", "calculate_total"}, + }, + "by_simple_name": { + "calculate_total": {"calculate_total"}, + "get_tax_amount": {"get_tax_amount"}, + "process_order": {"process_order"}, + }, + } + + def test_extract_definitions(self): + source_code = """ +class OrderManager: + def __init__(self, order_id): + self.order_id = order_id + + def process_payment(self): + print("Processing...") + +def calculate_discount(price): + return price * 0.10 + +class InventoryItem: + pass +""" + tree, _ = parse_code_to_ast(source_code, "Python") + functions = extract_definitions(tree, "Python", kinds=("functions",)) + assert ( + len(functions) == 3 + ) # '__init__', 'process_payment', and 'calculate_discount' + + assert functions[0].type == "function_definition" + first_func_text = functions[0].text.decode("utf-8") + assert "def __init__" in first_func_text + + classes = extract_definitions(tree, "Python", kinds=("classes",)) + assert len(classes) == 2 # OrderManager, InventoryItem + second_class_text = classes[1].text.decode("utf-8") + assert "class InventoryItem" in second_class_text + + def test_extract_definitions_empty(self): + tree, _ = parse_code_to_ast("", "Python") + assert extract_definitions(tree, "Python", kinds=("functions",)) == [] + assert extract_definitions(tree, "Python", kinds=("functions",)) == [] + assert extract_definitions(None, "Python", kinds=("classes",)) == [] + assert extract_definitions(None, "Python", kinds=("classes",)) == [] + + def test_get_qualified_name_functions(self): + source_code = """ +class CoreService: + class Validator: + def validate_payload(self, data): + return True + +def global_utility(): + pass + """ + + tree, _ = parse_code_to_ast(source_code, "Python") + index = collect_definitions(tree.root_node, "Python") + + functions = extract_definitions(tree, "Python", kinds=("functions",)) + assert len(functions) == 2 + + outer_function_name = qualified_name_from_index(functions[0], index) + inner_function_name = qualified_name_from_index(functions[1], index) + + assert outer_function_name == "CoreService.Validator.validate_payload" + assert inner_function_name == "global_utility" + + def test_get_qualified_classes(self): + source_code = """ +class FleetManagement: + class DroneController: + pass + """ + tree, _ = parse_code_to_ast(source_code, "Python") + index = collect_definitions(tree.root_node, "Python") + + classes = extract_definitions(tree, "Python", kinds=("classes",)) + assert len(classes) == 2 + + outer_class_name = qualified_name_from_index(classes[0], index) + inner_class_name = qualified_name_from_index(classes[1], index) + + assert outer_class_name == "FleetManagement" + assert inner_class_name == "FleetManagement.DroneController" + + def test_classify_reachability(self): + assert classify_reachability(None) == ReachabilityStatus.NOT_REACHABLE + assert classify_reachability({}) == ReachabilityStatus.NOT_REACHABLE + assert ( + classify_reachability( + {"sym1": {"exact_match_fingerprint": "hash123", "called": True}} + ) + == ReachabilityStatus.REACHABLE + ) + + assert ( + classify_reachability( + { + "sym1": { + "called": True, + "reachable_from": ["main_function", "api_handler"], + } + } + ) + == ReachabilityStatus.REACHABLE + ) + assert ( + classify_reachability({"sym1": {"defined": True, "called": False}}) + == ReachabilityStatus.POTENTIALLY_REACHABLE + ) + assert ( + classify_reachability( + {"sym1": {"exact_match_fingerprint": "hash123", "called": False}} + ) + == ReachabilityStatus.POTENTIALLY_REACHABLE + ) + assert ( + classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}) + == ReachabilityStatus.NOT_REACHABLE + ) From 3799817fd19aad3b9e6f1e4ef0d76ae443e81e68 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 5 Jun 2026 11:03:43 +0300 Subject: [PATCH 02/50] Add more test for reachability and remove redundant code Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 67 ++----- scanpipe/pipes/symbols.py | 54 +++++- .../tests/pipes/test_symbols_reachability.py | 175 +++++++++++++++++- 3 files changed, 236 insertions(+), 60 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 5531a1f495..5169eba94d 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -39,6 +39,7 @@ from scanpipe.pipes.symbols import extract_calls_in_node from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols +from scanpipe.pipes.symbols import is_nested_function from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -219,52 +220,6 @@ def get_changed_lines(diff_text, file_path): return removed, added -def query_captures(language, kind, node): - """ - Re-run a definition query on the root of `node`'s tree so ancestors can - be compared. Query caching is handled by `scanpipe.pipes.symbols`. - """ - from scanpipe.pipes.symbols import get_query - from scanpipe.pipes.symbols import run_query - - root = node - - while root.parent is not None: - root = root.parent - - query = get_query(language, kind) - return list(run_query(query, root)) - - -def is_nested_function(node, language): - function_nodes = { - captured_node - for captured_node, _ in query_captures(language, "functions", node) - } - class_nodes = { - captured_node for captured_node, _ in query_captures(language, "classes", node) - } - - if node not in function_nodes: - return False - - function_types = {captured_node.type for captured_node in function_nodes} - class_types = {captured_node.type for captured_node in class_nodes} - - parent = node.parent - - while parent is not None: - if parent.type in function_types: - return True - - if parent.type in class_types: - return False - - parent = parent.parent - - return False - - def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -506,7 +461,7 @@ def collect_and_store_symbol_reachability_results(project, logger=None): append_symbol_reachability_result(resource, result) except Exception as e: - logger.exception( + logger( "Failed to collect symbol reachability for " f"{vcs_url}@{commit_hash}: {e}" ) @@ -515,24 +470,36 @@ def collect_and_store_symbol_reachability_results(project, logger=None): def compute_reachable_symbols(call_graph, target_simple_names): + """ + Find all symbols that can transitively reach any of ``target_simple_names``. + + Reachability is matched on *simple* names (the call graph records callee + tokens, not fully-qualified names), so distinct symbols sharing a name are + treated as equivalent. This can over-approximate reachability. + + Returns: + (reachable_callers, has_direct_call) + reachable_callers: qualified names of all transitive callers + has_direct_call: whether any symbol calls a target directly + + """ if not call_graph or not target_simple_names: return set(), False edges = call_graph["edges"] targets = set(target_simple_names) - callers_of = {} + callers_of: dict[str, set[str]] = {} for caller_qn, callees in edges.items(): for callee_simple in callees: callers_of.setdefault(callee_simple, set()).add(caller_qn) - direct_callers = set() + direct_callers: set[str] = set() for target in targets: direct_callers |= callers_of.get(target, set()) has_direct_call = bool(direct_callers) - by_simple = call_graph["by_simple_name"] qn_to_simple = {qn: meta["simple_name"] for qn, meta in call_graph["nodes"].items()} reachable = set(direct_callers) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index c6247dbc44..cac76562e3 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -207,6 +207,7 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): }, } + @cache def load_language(language: str) -> Language: if language not in TS_LANGUAGE_WHEELS: @@ -256,7 +257,48 @@ def run_query(query: Query, root_node): yield def_nodes[0], name -def extract_calls_in_node(node, language: str) -> set[str]: +def query_captures(language, kind, node): + """Re-run a definition query on the root of node's tree.""" + query = get_query(language, kind) + return list(run_query(query, _root_of(node))) + + +def _root_of(node): + while node.parent is not None: + node = node.parent + return node + + +def is_nested_function(node, language): + function_nodes = { + captured_node + for captured_node, _ in query_captures(language, "functions", node) + } + class_nodes = { + captured_node for captured_node, _ in query_captures(language, "classes", node) + } + + if node not in function_nodes: + return False + + function_types = {captured_node.type for captured_node in function_nodes} + class_types = {captured_node.type for captured_node in class_nodes} + + parent = node.parent + + while parent is not None: + if parent.type in function_types: + return True + + if parent.type in class_types: + return False + + parent = parent.parent + + return False + + +def extract_calls_in_node(node, language: str): query = get_query(language, "calls") if query is None or node is None: return set() @@ -270,7 +312,7 @@ def extract_calls_in_node(node, language: str) -> set[str]: return names -def collect_definitions(root_node, language: str) -> dict[int, dict]: +def collect_definitions(root_node, language: str): index: dict[int, dict] = {} for kind in ("functions", "classes"): query = get_query(language, kind) @@ -311,13 +353,7 @@ def extract_symbols(tree, changed_lines: list[int], language: str): return enclosing -def _root_of(node): - while node.parent is not None: - node = node.parent - return node - - -def qualified_name_from_index(node, index: dict[int, dict]) -> str: +def qualified_name_from_index(node, index): parts = [] curr = node while curr is not None: diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 30d4c924dd..1a5fd7b3ad 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -30,9 +30,12 @@ from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_call_graph +from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results -from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.reachability import diff_changed_symbols +from scanpipe.pipes.reachability import get_changed_lines +from scanpipe.pipes.symbols import collect_definitions, extract_symbols from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -291,3 +294,173 @@ def test_classify_reachability(self): classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}) == ReachabilityStatus.NOT_REACHABLE ) + + def test_get_changed_lines(self): + data = Path(__file__).parent.parent / "data" / "reachability" + diff_text = (data / "diff-app.patch").read_text(encoding="utf-8") + + removed, added = get_changed_lines(diff_text, "app.py") + assert removed == [17, 18, 19, 24] + assert added == [17, 18, 19, 20, 21, 22, 27, 28, 29, 30] + + def test_build_symbol_metadata_processing(self): + source_code = """ +class Controller: + def process_data(payload): + def inner_helper(): + return True + return payload.strip() + +if True: + def process_data(payload): + return payload +""" + tree, _ = parse_code_to_ast(source_code, "Python") + nodes = extract_definitions(tree, "Python", kinds=("functions",)) + + metadata = build_symbol_metadata(nodes, "Python") + assert metadata == { + "Controller.process_data": { + "qualified_name": "Controller.process_data", + "simple_name": "process_data", + "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", + "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", + "start_line": 3, + "end_line": 6, + "node_type": "function_definition", + }, + "process_data": { + "qualified_name": "process_data", + "simple_name": "process_data", + "text": "def process_data(payload):\n return payload", + "fingerprint": "000000022020300e882a900807880d0300010000", + "start_line": 9, + "end_line": 10, + "node_type": "function_definition", + }, + } + + def test_diff_changed_symbols(self): + vuln_meta = { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n return os.path.join(base, filename)", + }, + "sanitize_input": { + "qualified_name": "app.sanitize_input", + "text": "def sanitize_input(x):\n return x.strip()", + }, + "deprecated_logger": { + "qualified_name": "app.deprecated_logger", + "text": "def deprecated_logger():\n print('legacy')", + }, + } + + fixed_meta = { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + }, + "sanitize_input": { + "qualified_name": "app.sanitize_input", + "text": "def sanitize_input(x):\n return x.strip()", + }, + "audit_trail": { + "qualified_name": "app.audit_trail", + "text": "def audit_trail():\n log.info('action')", + }, + } + + vuln_only, fixed_only = diff_changed_symbols(vuln_meta, fixed_meta) + + assert vuln_only == { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n return os.path.join(base, filename)", + }, + "deprecated_logger": { + "qualified_name": "app.deprecated_logger", + "text": "def deprecated_logger():\n print('legacy')", + }, + } + assert fixed_only == { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + }, + "audit_trail": { + "qualified_name": "app.audit_trail", + "text": "def audit_trail():\n log.info('action')", + }, + } + + def test_analyze_patched_file(self): + vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") + fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") + diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") + + vuln_meta, fixed_meta, lang = analyze_patched_file( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + diff_text=diff_text, + file_path="app.py", + ) + + assert vuln_meta == { + "serve_report": { + "qualified_name": "serve_report", + "simple_name": "serve_report", + "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # VULNERABLE: Direct concatenation allows Path Traversal\n # An attacker passing "../../etc/passwd" could read system files.\n return os.path.join(generator.base_dir, filename)\n\n if not requested_file:\n return "Error: No file specified"\n\n target_path = build_file_path(requested_file)\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', + "fingerprint": "000000556d322a47595af353274b000aa324e014", + "start_line": 11, + "end_line": 30, + "node_type": "function_definition", + } + } + assert fixed_meta == { + "serve_report": { + "qualified_name": "serve_report", + "simple_name": "serve_report", + "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', + "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", + "start_line": 11, + "end_line": 36, + "node_type": "function_definition", + } + } + + def test_extract_symbols(self): + source_code = ( + "def serve_report(request):\n" # Line 1 (Row 0) + " # Some processing here\n" # Line 2 (Row 1) + " def build_path(filename):\n" # Line 3 (Row 2) + " return filename.strip()\n" # Line 4 (Row 3) <- Targeted Change + " return build_path(request)\n" # Line 5 (Row 4) + ) + + tree, _ = parse_code_to_ast(source_code, "Python") + + changed_lines = [4] + enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + + assert len(enclosing_symbols) == 1 + target_node = enclosing_symbols[0] + assert target_node.type == "function_definition" + + node_text = target_node.text.decode("utf-8") + assert "def build_path" in node_text + assert "def serve_report" not in node_text + + def test_extract_symbols_deduplication(self): + source_code = ( + "def calculate_total(price, tax):\n" + " amount = price * tax\n" # Line 2 -> Changed + " return price + amount\n" # Line 3 -> Changed + ) + + tree, _ = parse_code_to_ast(source_code, "Python") + changed_lines = [2, 3] + + enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + assert len(enclosing_symbols) == 1 + assert enclosing_symbols[0].type == "function_definition" \ No newline at end of file From a3d1172c01d433b2186f9b1d8a5e4b9913de687b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 10 Jun 2026 15:08:45 +0300 Subject: [PATCH 03/50] Fix the format bugs and refactor the code Signed-off-by: ziad hany --- .../pipelines/collect_symbols_reachability.py | 8 +- scanpipe/pipes/reachability.py | 394 +++++++++++------- scanpipe/pipes/symbols.py | 14 +- scanpipe/tests/data/reachability/app.py | 2 +- scanpipe/tests/data/reachability/fixed-app.py | 2 +- scanpipe/tests/data/reachability/vuln-app.py | 2 +- .../tests/pipes/test_symbols_reachability.py | 381 +++++++++-------- 7 files changed, 473 insertions(+), 330 deletions(-) diff --git a/scanpipe/pipelines/collect_symbols_reachability.py b/scanpipe/pipelines/collect_symbols_reachability.py index 15519fc661..c1d5fb11c4 100644 --- a/scanpipe/pipelines/collect_symbols_reachability.py +++ b/scanpipe/pipelines/collect_symbols_reachability.py @@ -12,9 +12,7 @@ class SymbolReachability(Pipeline): - """ - Patch reachability analysis, for given a vulnerability patches - """ + """Patch reachability analysis for given vulnerability patches.""" download_inputs = False is_addon = True @@ -26,8 +24,8 @@ def steps(cls): def analyze_and_store_symbol_reachability(self): """ - Perform symbol-level reachability analysis for each patch. - This step compares the AST of patched/vulnerable files against the codebase resources. + Perform symbol-level reachability analysis for each patch. This step compares + the AST of patched/vulnerable files against the codebase resources. Results are stored directly in the 'extra_data' of each CodebaseResource. """ reachability.collect_and_store_symbol_reachability_results( diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 5169eba94d..e9ebd97378 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -29,16 +29,16 @@ from git import Repo from git.diff import NULL_TREE from git.exc import BadName -from matchcode_toolkit.fingerprinting import create_file_fingerprints from scancode.api import get_file_info from unidiff import PatchSet from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import _root_of from scanpipe.pipes.symbols import collect_definitions -from scanpipe.pipes.symbols import extract_calls_in_node +from scanpipe.pipes.symbols import create_exact_symbol_fingerprint from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols +from scanpipe.pipes.symbols import get_query from scanpipe.pipes.symbols import is_nested_function from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -53,14 +53,16 @@ class ReachabilityStatus(str, Enum): def api_mocker(): - """ - TODO: Remove this once the API patch url is done - """ + """TODO: Remove this once the API patch url is done""" return [ { "vcs_url": "https://github.com/pallets/flask", "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", }, + # { + # "vcs_url": "https://github.com/aio-libs/aiohttp", + # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", + # } ] @@ -100,7 +102,7 @@ def normalize_text(content): def is_supported_language(language): - """A language is supported if we have tree-sitter queries for it.""" + """Return True if the language is supported by tree-sitter queries.""" return bool(language) and language in TS_QUERIES @@ -223,28 +225,18 @@ def get_changed_lines(diff_text, file_path): def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed - versions. Pair by qualified name first. + versions. Utilizes the unique suffix keys generated by build_symbol_metadata. """ - fixed_by_qn = { - metadata["qualified_name"]: metadata for metadata in fixed_meta.values() - } - - vuln_by_qn = { - metadata["qualified_name"]: metadata for metadata in vuln_meta.values() - } - vuln_only = { key: metadata for key, metadata in vuln_meta.items() - if fixed_by_qn.get(metadata["qualified_name"], {}).get("text") - != metadata["text"] + if fixed_meta.get(key, {}).get("text") != metadata["text"] } fixed_only = { key: metadata for key, metadata in fixed_meta.items() - if vuln_by_qn.get(metadata["qualified_name"], {}).get("text") - != metadata["text"] + if vuln_meta.get(key, {}).get("text") != metadata["text"] } return vuln_only, fixed_only @@ -313,9 +305,6 @@ def collect_patch_symbols(repo, commit_hash): ... } - Symbols are bucketed by language so resources are only matched against - patch symbols extracted from the same language. - """ commit, parent = get_commit_and_parent(repo, commit_hash) diff_text = get_commit_diff_text(repo, parent, commit) @@ -379,11 +368,6 @@ def collect_and_store_symbol_reachability_results(project, logger=None): For each known patch commit, determine whether each project codebase resource is reachable to the vulnerable code by comparing tree-sitter ASTs of the patch versus the resource. - - Result classification: - - REACHABLE - - POTENTIALLY_REACHABLE - - NOT_REACHABLE """ candidate_resources = project.codebaseresources.files().filter( is_binary=False, @@ -394,10 +378,10 @@ def collect_and_store_symbol_reachability_results(project, logger=None): for patch in api_mocker(): vcs_url = patch["vcs_url"] commit_hash = patch["commit_hash"] - repo_path = None try: - repo_path = clone_repo(vcs_url, commit_hash) - repo = Repo(repo_path) + # repo_path = clone_repo(vcs_url, commit_hash) + # repo = Repo("/home/ziad-hany/PycharmProjects/flask/") + repo = Repo("/home/ziad-hany/PycharmProjects/aiohttp") patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) @@ -406,18 +390,16 @@ def collect_and_store_symbol_reachability_results(project, logger=None): for resource in candidate_resources: resource_language = resource.programming_language - if resource_language not in patch_symbols_by_language: continue - resource_text = normalize_text(resource.file_content) - + resource_text = resource.file_content if not resource_text: continue patch_symbols = patch_symbols_by_language[resource_language] - vuln_metadata = patch_symbols["vulnerable"] - fixed_metadata = patch_symbols["fixed"] + vuln_patch_metadata = patch_symbols["vulnerable"] + fixed_patch_metadata = patch_symbols["fixed"] resource_index = build_resource_index( resource_text, @@ -428,12 +410,12 @@ def collect_and_store_symbol_reachability_results(project, logger=None): continue vuln_match_symbols = match_symbols_against_resource( - vuln_metadata, + vuln_patch_metadata, resource_index, ) fixed_match_symbols = match_symbols_against_resource( - fixed_metadata, + fixed_patch_metadata, resource_index, ) @@ -443,84 +425,36 @@ def collect_and_store_symbol_reachability_results(project, logger=None): result = { "reachability_status": classify_reachability(vuln_match_symbols), "summary": { - "vulnerable_symbols": sorted(vuln_match_symbols), - "fixed_symbols": sorted(fixed_match_symbols), "call_paths": { - qn: ev.get("reachable_from", []) - for qn, ev in vuln_match_symbols.items() + qualified_name: ev.get("reachable_from", []) + for qualified_name, ev in vuln_match_symbols.items() if ev.get("called") }, }, "evidence": vuln_match_symbols, + "vulnerable_symbols": sorted(vuln_match_symbols), + "fixed_symbols": sorted(fixed_match_symbols), "patch": { "vcs_url": vcs_url, "commit_hash": commit_hash, }, } - + print(result) append_symbol_reachability_result(resource, result) except Exception as e: logger( - "Failed to collect symbol reachability for " + f"Failed to collect symbol reachability for " f"{vcs_url}@{commit_hash}: {e}" ) finally: - cleanup_repo(repo_path) - - -def compute_reachable_symbols(call_graph, target_simple_names): - """ - Find all symbols that can transitively reach any of ``target_simple_names``. - - Reachability is matched on *simple* names (the call graph records callee - tokens, not fully-qualified names), so distinct symbols sharing a name are - treated as equivalent. This can over-approximate reachability. - - Returns: - (reachable_callers, has_direct_call) - reachable_callers: qualified names of all transitive callers - has_direct_call: whether any symbol calls a target directly - - """ - if not call_graph or not target_simple_names: - return set(), False - - edges = call_graph["edges"] - targets = set(target_simple_names) - - callers_of: dict[str, set[str]] = {} - for caller_qn, callees in edges.items(): - for callee_simple in callees: - callers_of.setdefault(callee_simple, set()).add(caller_qn) - - direct_callers: set[str] = set() - for target in targets: - direct_callers |= callers_of.get(target, set()) - - has_direct_call = bool(direct_callers) - - qn_to_simple = {qn: meta["simple_name"] for qn, meta in call_graph["nodes"].items()} - - reachable = set(direct_callers) - frontier = list(direct_callers) - - while frontier: - current_qn = frontier.pop() - current_simple = qn_to_simple.get(current_qn) - if not current_simple: - continue - for parent_qn in callers_of.get(current_simple, ()): - if parent_qn not in reachable: - reachable.add(parent_qn) - frontier.append(parent_qn) + if repo: + repo.close() - return reachable, has_direct_call + # cleanup_repo(repo_path) def build_resource_index(resource_text, language): - resource_text = normalize_text(resource_text) - if not is_supported_language(language) or not resource_text: return None @@ -551,34 +485,38 @@ def build_resource_index(resource_text, language): } -def match_symbols_against_resource(symbols, resource_index): - if not symbols or not resource_index: +def match_symbols_against_resource(patch_symbols_metadata, resource_index): + if not patch_symbols_metadata or not resource_index: return {} call_graph = resource_index.get("call_graph") - target_simple_names = {metadata["simple_name"] for metadata in symbols.values()} - reachable_callers, _has_direct = compute_reachable_symbols( + target_qualified_names = { + metadata["qualified_name"] for metadata in patch_symbols_metadata.values() + } + + reachable_callers, _ = compute_reachable_symbols( call_graph, - target_simple_names, + target_qualified_names, ) - called_simple_names = set() + called_qualified_names = set() + if call_graph: - for callees in call_graph["edges"].values(): - called_simple_names |= callees + for callees in call_graph.get("edges_qualified", {}).values(): + called_qualified_names |= set(callees) matched = {} - for metadata in symbols.values(): + + for metadata in patch_symbols_metadata.values(): qualified_name = metadata["qualified_name"] - simple_name = metadata["simple_name"] fingerprint = metadata["fingerprint"] - defined = qualified_name in resource_index["definitions"] + defined = qualified_name in resource_index.get("definitions", {}) fingerprint_hit = bool( - fingerprint and fingerprint in resource_index["fingerprints"] + fingerprint and fingerprint in resource_index.get("fingerprints", {}) ) - called = simple_name in called_simple_names + called = qualified_name in called_qualified_names if not (defined or fingerprint_hit or called): continue @@ -608,12 +546,6 @@ def classify_reachability(evidence): if not evidence: return ReachabilityStatus.NOT_REACHABLE - SEVERITY_RANK = { - ReachabilityStatus.NOT_REACHABLE: 0, - ReachabilityStatus.POTENTIALLY_REACHABLE: 1, - ReachabilityStatus.REACHABLE: 2, - } - highest_status = ReachabilityStatus.NOT_REACHABLE for item in evidence.values(): @@ -622,17 +554,12 @@ def classify_reachability(evidence): is_exact = "exact_match_fingerprint" in item is_defined = bool(item.get("defined")) - if is_called or (has_path and is_exact): + if is_called or has_path: return ReachabilityStatus.REACHABLE - elif has_path or is_exact or is_defined: - current_item_status = ReachabilityStatus.POTENTIALLY_REACHABLE - - else: - current_item_status = ReachabilityStatus.NOT_REACHABLE + if is_exact or is_defined: + highest_status = ReachabilityStatus.POTENTIALLY_REACHABLE - if SEVERITY_RANK[current_item_status] > SEVERITY_RANK[highest_status]: - highest_status = current_item_status return highest_status @@ -650,7 +577,7 @@ def build_symbol_metadata(nodes, language, index=None): continue body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_file_fingerprints(content=body_text) or {} + fingerprints = create_exact_symbol_fingerprint(body_text) key = qualified_name suffix = 1 @@ -660,9 +587,8 @@ def build_symbol_metadata(nodes, language, index=None): metadata[key] = { "qualified_name": qualified_name, - "simple_name": qualified_name.rsplit(".", 1)[-1], "text": body_text, - "fingerprint": fingerprints.get("halo1"), + "fingerprint": fingerprints, "start_line": node.start_point[0] + 1, "end_line": node.end_point[0] + 1, "node_type": node.type, @@ -675,23 +601,211 @@ def build_call_graph(tree, language): return None index = collect_definitions(tree.root_node, language) - definition_nodes = [d["node"] for d in index.values()] - metadata = build_symbol_metadata(definition_nodes, language, index=index) - qualified_name_to_node = {} - for node in definition_nodes: + graph_meta = {} + for definition in index.values(): + node = definition["node"] qualified_name = qualified_name_from_index(node, index) - if qualified_name: - qualified_name_to_node.setdefault(qualified_name, node) - - edges = {} - by_simple_name = {} - for qualified_name, meta in metadata.items(): - canonical = meta["qualified_name"] - node = qualified_name_to_node.get(canonical) - if node is None: + + if not qualified_name: continue - edges.setdefault(canonical, set()).update(extract_calls_in_node(node, language)) - by_simple_name.setdefault(meta["simple_name"], set()).add(canonical) - return {"nodes": metadata, "edges": edges, "by_simple_name": by_simple_name} + body_text = node.text.decode("utf-8", errors="replace") + fingerprints = create_exact_symbol_fingerprint(body_text) or {} + + graph_meta[qualified_name] = { + "qualified_name": qualified_name, + "node": node, + "node_type": node.type, + "fingerprint": fingerprints, + } + + definitions_by_name = {} + class_methods = set() + + for qualified_name, metadata in graph_meta.items(): + name = qualified_name.rsplit(".", 1)[-1] + definitions_by_name.setdefault(name, set()).add(qualified_name) + + if metadata["node_type"] == "function_definition" and "." in qualified_name: + class_methods.add(qualified_name) + + edges_qualified = {} + for qualified_name, metadata in graph_meta.items(): + direct_calls = extract_direct_calls(metadata["node"], language, index) + + resolved_callees = set() + + for receiver_name, callee_name in direct_calls: + resolved_callees |= resolve_callee( + receiver_name=receiver_name, + callee_name=callee_name, + owner_qn=qualified_name, + definitions_by_name=definitions_by_name, + class_methods=class_methods, + ) + + edges_qualified[qualified_name] = resolved_callees + + return { + "nodes": graph_meta, + "edges_qualified": edges_qualified, + } + + +def extract_direct_calls(node, language, definition_index): + """ + Return direct calls inside `node`, excluding calls inside nested definitions. + + Returns: + list of (receiver_name, callee_name) + + Examples: + foo() -> (None, "foo") + self.foo() -> ("self", "foo") + obj.foo() -> ("obj", "foo") + + """ + query = get_query(language, "calls") + if query is None or node is None: + return [] + + definition_ids = set(definition_index) + calls = [] + + for _, captures in query.matches(node): + for callee_node in captures.get("callee", []): + if is_inside_nested_definition( + node=callee_node, + owner_node=node, + definition_ids=definition_ids, + ): + continue + + receiver_name = get_call_receiver(callee_node) + callee_name = node_text(callee_node) + + if callee_name: + calls.append((receiver_name, callee_name)) + + return calls + + +def is_inside_nested_definition(node, owner_node, definition_ids): + """ + Return True if node is inside a nested function/class within `owner_node`. + + Example: + def outer(): + foo() # belongs to outer + + def inner(): + bar() # nested; should not count as outer's call + + """ + current = node.parent + + while current is not None and current is not owner_node: + if current.id in definition_ids: + return True + + current = current.parent + + return False + + +def node_text(node): + return node.text.decode("utf-8", errors="replace") + + +def get_call_receiver(callee_node): + """ + Return receiver name for attribute calls. + + Examples: + foo() -> None + self.foo() -> "self" + obj.foo() -> "obj" + + """ + parent = callee_node.parent + + if parent is None or parent.type != "attribute": + return None + + object_node = parent.child_by_field_name("object") + if object_node is None: + return None + + return node_text(object_node) + + +def resolve_callee( + receiver_name, callee_name, owner_qn, definitions_by_name, class_methods +): + """ + Resolve a call to candidate qualified names. + + Examples: + self.foo() from class A -> {"A.foo"} if A.foo exists + foo() -> definitions named "foo" + + """ + if receiver_name == "self": + owner_class = get_owner_class_name(owner_qn) + + if owner_class: + method_qn = f"{owner_class}.{callee_name}" + + if method_qn in class_methods: + return {method_qn} + + candidates = definitions_by_name.get(callee_name, set()) + return set(candidates) + + +def get_owner_class_name(owner_qn): + """ + Return enclosing class name from a qualified name. + + Examples: + "User.save" -> "User" + "User.Inner.save" -> "User.Inner" + "save" -> None + + """ + if "." not in owner_qn: + return None + + return owner_qn.rsplit(".", 1)[0] + + +def compute_reachable_symbols(call_graph, target_qualified_names): + """Transitive callers using resolved qualified-name edges.""" + if not call_graph or not target_qualified_names: + return set(), False + + edges = call_graph.get("edges_qualified") + if not edges: + return set(), False + + callers_of = {} + for caller, callees in edges.items(): + for callee in callees: + callers_of.setdefault(callee, set()).add(caller) + + targets = set(target_qualified_names) + direct = set() + for target in targets: + direct |= callers_of.get(target, set()) + + reachable = set(direct) + frontier = list(direct) + while frontier: + cur = frontier.pop() + for parent in callers_of.get(cur, ()): + if parent not in reachable: + reachable.add(parent) + frontier.append(parent) + + return reachable, bool(direct) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index cac76562e3..da6f359f65 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -20,6 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import hashlib import importlib from functools import cache @@ -191,7 +192,6 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "pygments": symbols_pygments.get_pygments_symbols, } -# https://github.com/Aider-AI/aider/tree/5dc9490bb35f9729ef2c95d00a19ccd30c26339c/aider/queries/tree-sitter-language-pack TS_QUERIES = { "Python": { "functions": """ @@ -202,7 +202,9 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): """, "calls": """ (call function: (identifier) @callee) - (call function: (attribute attribute: (identifier) @callee)) + (call function: (attribute + object: (_) @receiver + attribute: (identifier) @callee)) """, }, } @@ -362,3 +364,11 @@ def qualified_name_from_index(node, index): parts.append(definition["name"]) curr = curr.parent return ".".join(reversed(parts)) + + +def create_exact_symbol_fingerprint(text): + if text is None: + return None + + text = text.encode("utf-8", errors="replace") + return hashlib.sha256(text).hexdigest() diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/app.py index c64ae7d9d1..b8c9eff5e0 100644 --- a/scanpipe/tests/data/reachability/app.py +++ b/scanpipe/tests/data/reachability/app.py @@ -31,5 +31,5 @@ def build_file_path(filename): def unrelated_top_level_function(): - """An extra function to test AST node boundaries.""" + """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/fixed-app.py index 3296bb843e..ca5a6f4c8b 100644 --- a/scanpipe/tests/data/reachability/fixed-app.py +++ b/scanpipe/tests/data/reachability/fixed-app.py @@ -37,5 +37,5 @@ def build_file_path(filename): def unrelated_top_level_function(): - """An extra function to test AST node boundaries.""" + """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/vuln-app.py index c64ae7d9d1..b8c9eff5e0 100644 --- a/scanpipe/tests/data/reachability/vuln-app.py +++ b/scanpipe/tests/data/reachability/vuln-app.py @@ -31,5 +31,5 @@ def build_file_path(filename): def unrelated_top_level_function(): - """An extra function to test AST node boundaries.""" + """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 1a5fd7b3ad..68bbe9b964 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -29,14 +29,15 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import analyze_patched_file -from scanpipe.pipes.reachability import build_call_graph from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols from scanpipe.pipes.reachability import get_changed_lines -from scanpipe.pipes.symbols import collect_definitions, extract_symbols +from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions +from scanpipe.pipes.symbols import extract_symbols from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -101,85 +102,31 @@ def test_collect_and_store_symbol_reachability_results( resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") - assert results == [ - { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "summary": { - "call_paths": {}, + self.assertEqual( + results, + [ + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "summary": {"call_paths": {}}, + "evidence": { + "serve_report": { + "called": False, + "defined": True, + "reachable_from": [], + "exact_match_fingerprint": ( + "e341b914f9823915e0685396a730d421ec9e3635" + ), + } + }, "fixed_symbols": ["serve_report"], "vulnerable_symbols": ["serve_report"], - }, - "evidence": { - "serve_report": { - "called": False, - "defined": True, - "reachable_from": [], - "exact_match_fingerprint": "000000556d322a47595af353274b000aa324e014", - } - }, - "reachability_status": "POTENTIALLY_REACHABLE", - } - ] - - def test_build_call_graph(self): - source_code = """ -def calculate_total(price, tax): - return price + get_tax_amount(price, tax) - -def get_tax_amount(price, tax): - return price * tax - -def process_order(): - total = calculate_total(100, 0.05) - print("Done") -""" - tree, _ = parse_code_to_ast(source_code, "Python") - result = build_call_graph(tree, "Python") - - assert result == { - "nodes": { - "calculate_total": { - "qualified_name": "calculate_total", - "simple_name": "calculate_total", - "text": "def calculate_total(price, tax):\n return price + get_tax_amount(price, tax)", - "fingerprint": "00000008060105fd3624134884412006ce880936", - "start_line": 2, - "end_line": 3, - "node_type": "function_definition", - }, - "get_tax_amount": { - "qualified_name": "get_tax_amount", - "simple_name": "get_tax_amount", - "text": "def get_tax_amount(price, tax):\n return price * tax", - "fingerprint": "000000058f0ee87d9669f20b1f473137b665bb20", - "start_line": 5, - "end_line": 6, - "node_type": "function_definition", - }, - "process_order": { - "qualified_name": "process_order", - "simple_name": "process_order", - "text": 'def process_order():\n total = calculate_total(100, 0.05)\n print("Done")', - "fingerprint": "000000071c3e6902da5c2b322386eff29068e3e2", - "start_line": 8, - "end_line": 10, - "node_type": "function_definition", - }, - }, - "edges": { - "calculate_total": {"get_tax_amount"}, - "get_tax_amount": set(), - "process_order": {"print", "calculate_total"}, - }, - "by_simple_name": { - "calculate_total": {"calculate_total"}, - "get_tax_amount": {"get_tax_amount"}, - "process_order": {"process_order"}, - }, - } + "reachability_status": "POTENTIALLY_REACHABLE", + } + ], + ) def test_extract_definitions(self): source_code = """ @@ -198,25 +145,25 @@ class InventoryItem: """ tree, _ = parse_code_to_ast(source_code, "Python") functions = extract_definitions(tree, "Python", kinds=("functions",)) - assert ( - len(functions) == 3 + self.assertEqual( + len(functions), 3 ) # '__init__', 'process_payment', and 'calculate_discount' - assert functions[0].type == "function_definition" + self.assertEqual(functions[0].type, "function_definition") first_func_text = functions[0].text.decode("utf-8") - assert "def __init__" in first_func_text + self.assertIn("def __init__", first_func_text) classes = extract_definitions(tree, "Python", kinds=("classes",)) - assert len(classes) == 2 # OrderManager, InventoryItem + self.assertEqual(len(classes), 2) second_class_text = classes[1].text.decode("utf-8") - assert "class InventoryItem" in second_class_text + self.assertIn("class InventoryItem", second_class_text) def test_extract_definitions_empty(self): tree, _ = parse_code_to_ast("", "Python") - assert extract_definitions(tree, "Python", kinds=("functions",)) == [] - assert extract_definitions(tree, "Python", kinds=("functions",)) == [] - assert extract_definitions(None, "Python", kinds=("classes",)) == [] - assert extract_definitions(None, "Python", kinds=("classes",)) == [] + self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) + self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) + self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) + self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) def test_get_qualified_name_functions(self): source_code = """ @@ -233,13 +180,13 @@ def global_utility(): index = collect_definitions(tree.root_node, "Python") functions = extract_definitions(tree, "Python", kinds=("functions",)) - assert len(functions) == 2 + self.assertEqual(len(functions), 2) outer_function_name = qualified_name_from_index(functions[0], index) inner_function_name = qualified_name_from_index(functions[1], index) - assert outer_function_name == "CoreService.Validator.validate_payload" - assert inner_function_name == "global_utility" + self.assertEqual(outer_function_name, "CoreService.Validator.validate_payload") + self.assertEqual(inner_function_name, "global_utility") def test_get_qualified_classes(self): source_code = """ @@ -251,25 +198,25 @@ class DroneController: index = collect_definitions(tree.root_node, "Python") classes = extract_definitions(tree, "Python", kinds=("classes",)) - assert len(classes) == 2 + self.assertEqual(len(classes), 2) outer_class_name = qualified_name_from_index(classes[0], index) inner_class_name = qualified_name_from_index(classes[1], index) - assert outer_class_name == "FleetManagement" - assert inner_class_name == "FleetManagement.DroneController" + self.assertEqual(outer_class_name, "FleetManagement") + self.assertEqual(inner_class_name, "FleetManagement.DroneController") def test_classify_reachability(self): - assert classify_reachability(None) == ReachabilityStatus.NOT_REACHABLE - assert classify_reachability({}) == ReachabilityStatus.NOT_REACHABLE - assert ( + self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) + self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) + self.assertEqual( classify_reachability( {"sym1": {"exact_match_fingerprint": "hash123", "called": True}} - ) - == ReachabilityStatus.REACHABLE + ), + ReachabilityStatus.REACHABLE, ) - assert ( + self.assertEqual( classify_reachability( { "sym1": { @@ -277,22 +224,22 @@ def test_classify_reachability(self): "reachable_from": ["main_function", "api_handler"], } } - ) - == ReachabilityStatus.REACHABLE + ), + ReachabilityStatus.REACHABLE, ) - assert ( - classify_reachability({"sym1": {"defined": True, "called": False}}) - == ReachabilityStatus.POTENTIALLY_REACHABLE + self.assertEqual( + classify_reachability({"sym1": {"defined": True, "called": False}}), + ReachabilityStatus.POTENTIALLY_REACHABLE, ) - assert ( + self.assertEqual( classify_reachability( {"sym1": {"exact_match_fingerprint": "hash123", "called": False}} - ) - == ReachabilityStatus.POTENTIALLY_REACHABLE + ), + ReachabilityStatus.POTENTIALLY_REACHABLE, ) - assert ( - classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}) - == ReachabilityStatus.NOT_REACHABLE + self.assertEqual( + classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}), + ReachabilityStatus.NOT_REACHABLE, ) def test_get_changed_lines(self): @@ -300,8 +247,8 @@ def test_get_changed_lines(self): diff_text = (data / "diff-app.patch").read_text(encoding="utf-8") removed, added = get_changed_lines(diff_text, "app.py") - assert removed == [17, 18, 19, 24] - assert added == [17, 18, 19, 20, 21, 22, 27, 28, 29, 30] + self.assertEqual(removed, [17, 18, 19, 24]) + self.assertEqual(added, [17, 18, 19, 20, 21, 22, 27, 28, 29, 30]) def test_build_symbol_metadata_processing(self): source_code = """ @@ -319,26 +266,30 @@ def process_data(payload): nodes = extract_definitions(tree, "Python", kinds=("functions",)) metadata = build_symbol_metadata(nodes, "Python") - assert metadata == { - "Controller.process_data": { - "qualified_name": "Controller.process_data", - "simple_name": "process_data", - "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", - "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", - "start_line": 3, - "end_line": 6, - "node_type": "function_definition", - }, - "process_data": { - "qualified_name": "process_data", - "simple_name": "process_data", - "text": "def process_data(payload):\n return payload", - "fingerprint": "000000022020300e882a900807880d0300010000", - "start_line": 9, - "end_line": 10, - "node_type": "function_definition", + self.assertEqual( + metadata, + { + "Controller.process_data": { + "qualified_name": "Controller.process_data", + "text": "def process_data(payload):\n" + " def inner_helper():\n" + " return True\n" + " return payload.strip()", + "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", + "start_line": 3, + "end_line": 6, + "node_type": "function_definition", + }, + "process_data": { + "qualified_name": "process_data", + "text": "def process_data(payload):\n return payload", + "fingerprint": "000000022020300e882a900807880d0300010000", + "start_line": 9, + "end_line": 10, + "node_type": "function_definition", + }, }, - } + ) def test_diff_changed_symbols(self): vuln_meta = { @@ -359,7 +310,10 @@ def test_diff_changed_symbols(self): fixed_meta = { "serve_report": { "qualified_name": "app.serve_report", - "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + "text": "def serve_report():\n " + " if not target.startswith(base): " + "raise ValueError\n " + " return target", }, "sanitize_input": { "qualified_name": "app.sanitize_input", @@ -373,26 +327,34 @@ def test_diff_changed_symbols(self): vuln_only, fixed_only = diff_changed_symbols(vuln_meta, fixed_meta) - assert vuln_only == { - "serve_report": { - "qualified_name": "app.serve_report", - "text": "def serve_report():\n return os.path.join(base, filename)", - }, - "deprecated_logger": { - "qualified_name": "app.deprecated_logger", - "text": "def deprecated_logger():\n print('legacy')", - }, - } - assert fixed_only == { - "serve_report": { - "qualified_name": "app.serve_report", - "text": "def serve_report():\n if not target.startswith(base): raise ValueError\n return target", + self.assertEqual( + vuln_only, + { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n " + " return os.path.join(base, filename)", + }, + "deprecated_logger": { + "qualified_name": "app.deprecated_logger", + "text": "def deprecated_logger():\n print('legacy')", + }, }, - "audit_trail": { - "qualified_name": "app.audit_trail", - "text": "def audit_trail():\n log.info('action')", + ) + self.assertEqual( + fixed_only, + { + "serve_report": { + "qualified_name": "app.serve_report", + "text": "def serve_report():\n if not target.startswith(base): " + "raise ValueError\n return target", + }, + "audit_trail": { + "qualified_name": "app.audit_trail", + "text": "def audit_trail():\n log.info('action')", + }, }, - } + ) def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") @@ -406,28 +368,70 @@ def test_analyze_patched_file(self): file_path="app.py", ) - assert vuln_meta == { - "serve_report": { - "qualified_name": "serve_report", - "simple_name": "serve_report", - "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # VULNERABLE: Direct concatenation allows Path Traversal\n # An attacker passing "../../etc/passwd" could read system files.\n return os.path.join(generator.base_dir, filename)\n\n if not requested_file:\n return "Error: No file specified"\n\n target_path = build_file_path(requested_file)\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', - "fingerprint": "000000556d322a47595af353274b000aa324e014", - "start_line": 11, - "end_line": 30, - "node_type": "function_definition", - } - } - assert fixed_meta == { - "serve_report": { - "qualified_name": "serve_report", - "simple_name": "serve_report", - "text": 'def serve_report(request_payload):\n """Top-level function handling a request."""\n generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', - "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", - "start_line": 11, - "end_line": 36, - "node_type": "function_definition", - } - } + self.assertEqual( + vuln_meta, + { + "serve_report": { + "qualified_name": "serve_report", + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n ' + ' generator = ReportGenerator("/var/reports")\n ' + ' requested_file = request_payload.get("file")\n\n ' + "# Helper function nested inside serve_report\n " + "def build_file_path(filename):\n " + " # VULNERABLE: Direct concatenation allows Path Traversal\n " + ' # An attacker passing "../../etc/passwd" ' + "could read system files.\n " + " return os.path.join(generator.base_dir, filename)\n\n " + " if not requested_file:\n " + ' return "Error: No file specified"\n\n ' + " target_path = build_file_path(requested_file)\n\n" + " " + " " + "if os.path.exists(target_path):\n" + " " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', + "fingerprint": "000000556d322a47595af353274b000aa324e014", + "start_line": 11, + "end_line": 30, + "node_type": "function_definition", + } + }, + ) + + self.assertEqual( + fixed_meta, + { + "serve_report": { + "qualified_name": "serve_report", + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n ' + ' generator = ReportGenerator("/var/reports")\n ' + ' requested_file = request_payload.get("file")\n\n ' + " # Helper function nested inside serve_report\n " + " def build_file_path(filename):\n " + " # FIXED: Validate that the resolved " + "path stays within the base_dir\n " + " base = os.path.abspath(generator.base_dir)\n " + " target = os.path.abspath(os.path.join(base, filename))\n " + " if not target.startswith(base):\n " + ' raise ValueError("Path Traversal Detected")\n ' + " return target\n\n if not requested_file:\n " + ' return "Error: No file specified"\n\n try:\n ' + " target_path = build_file_path(requested_file)\n " + " except ValueError:\n " + ' return "Error: Invalid path"\n\n' + " if os.path.exists(target_path):\n " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', + "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", + "start_line": 11, + "end_line": 36, + "node_type": "function_definition", + } + }, + ) def test_extract_symbols(self): source_code = ( @@ -443,13 +447,13 @@ def test_extract_symbols(self): changed_lines = [4] enclosing_symbols = extract_symbols(tree, changed_lines, "Python") - assert len(enclosing_symbols) == 1 + self.assertEqual(len(enclosing_symbols), 1) target_node = enclosing_symbols[0] - assert target_node.type == "function_definition" + self.assertEqual(target_node.type, "function_definition") node_text = target_node.text.decode("utf-8") - assert "def build_path" in node_text - assert "def serve_report" not in node_text + self.assertIn("def build_path", node_text) + self.assertNotIn("def serve_report", node_text) def test_extract_symbols_deduplication(self): source_code = ( @@ -462,5 +466,22 @@ def test_extract_symbols_deduplication(self): changed_lines = [2, 3] enclosing_symbols = extract_symbols(tree, changed_lines, "Python") - assert len(enclosing_symbols) == 1 - assert enclosing_symbols[0].type == "function_definition" \ No newline at end of file + self.assertEqual(len(enclosing_symbols), 1) + self.assertEqual(enclosing_symbols[0].type, "function_definition") + + def test_compute_reachable_symbols(self): + call_graph = { + "edges_qualified": { + "app.main": {"app.helper", "app.safe_func"}, + "app.helper": {"app.vuln_func"}, + "app.direct_caller": {"app.vuln_func"}, + "app.unrelated": {"app.safe_func"}, + } + } + + target_qns = ["app.vuln_func"] + reachable, has_direct = compute_reachable_symbols(call_graph, target_qns) + self.assertTrue(has_direct) + + expected_reachable = {"app.main", "app.helper", "app.direct_caller"} + self.assertEqual(reachable, expected_reachable) From 2f0dd8496d0a38b933ae8d5827ab21c69613441c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 11 Jun 2026 18:47:37 +0300 Subject: [PATCH 04/50] Fix a bug in the import-catching logic and add a test. Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 255 +++++++++++------- scanpipe/pipes/symbols.py | 15 ++ .../tests/pipes/test_symbols_reachability.py | 39 ++- 3 files changed, 215 insertions(+), 94 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index e9ebd97378..ad8a347e8f 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -55,14 +55,14 @@ class ReachabilityStatus(str, Enum): def api_mocker(): """TODO: Remove this once the API patch url is done""" return [ - { - "vcs_url": "https://github.com/pallets/flask", - "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", - }, # { - # "vcs_url": "https://github.com/aio-libs/aiohttp", - # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", - # } + # "vcs_url": "https://github.com/pallets/flask", + # "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", + # }, + { + "vcs_url": "https://github.com/aio-libs/aiohttp", + "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", + } ] @@ -139,7 +139,7 @@ def get_commit_and_parent(repo, commit_hash): def get_commit_diff_text(repo, parent_commit, commit): """Whole-commit unified diff (used to extract changed line numbers).""" base = parent_commit.hexsha if parent_commit else EMPTY_TREE_SHA - return repo.git.diff(base, commit.hexsha, unified=0) + return repo.git.diff(base, commit.hexsha, unified=3) def get_changed_files(parent_commit, commit): @@ -195,7 +195,13 @@ def get_changed_files(parent_commit, commit): def get_changed_lines(diff_text, file_path): - """Return `(removed_lines, added_lines)` for one file from a unified diff.""" + """ + Return `(removed_lines, added_lines)` for one file. + + For pure-insertion hunks (no removed lines) we anchor the vulnerable side + to the hunk's source location so the enclosing old symbol is still found. + For pure-deletion hunks we do the mirror image on the added side. + """ removed = [] added = [] @@ -208,20 +214,37 @@ def get_changed_lines(diff_text, file_path): (patched_file.source_file or "").removeprefix("a/"), (patched_file.target_file or "").removeprefix("b/"), } - if file_path not in candidates: continue for hunk in patched_file: - for line in hunk: - if line.is_removed and line.source_line_no: - removed.append(line.source_line_no) - elif line.is_added and line.target_line_no: - added.append(line.target_line_no) + hunk_removed = [ + line.source_line_no + for line in hunk + if line.is_removed and line.source_line_no + ] + hunk_added = [ + line.target_line_no + for line in hunk + if line.is_added and line.target_line_no + ] + + # Pure insertion: nothing removed -> anchor old side to the + # line just before the insertion point in the source file. + if hunk_added and not hunk_removed: + anchor = max(hunk.source_start, 1) + hunk_removed = [anchor] + + # Pure deletion: nothing added -> anchor new side similarly. + if hunk_removed and not hunk_added: + anchor = max(hunk.target_start, 1) + hunk_added = [anchor] + + removed.extend(hunk_removed) + added.extend(hunk_added) return removed, added - def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -343,26 +366,6 @@ def collect_patch_symbols(repo, commit_hash): return by_language -def append_symbol_reachability_result(resource, result): - """ - Append one symbol reachability result to the resource extra_data without - overwriting previous results. - """ - extra_data = resource.extra_data or {} - existing_results = extra_data.get("symbols_reachability", []) - - if not isinstance(existing_results, list): - existing_results = [existing_results] - - existing_results.append(result) - - resource.update_extra_data( - { - "symbols_reachability": existing_results, - } - ) - - def collect_and_store_symbol_reachability_results(project, logger=None): """ For each known patch commit, determine whether each project codebase @@ -398,9 +401,6 @@ def collect_and_store_symbol_reachability_results(project, logger=None): continue patch_symbols = patch_symbols_by_language[resource_language] - vuln_patch_metadata = patch_symbols["vulnerable"] - fixed_patch_metadata = patch_symbols["fixed"] - resource_index = build_resource_index( resource_text, resource_language, @@ -409,38 +409,34 @@ def collect_and_store_symbol_reachability_results(project, logger=None): if not resource_index: continue - vuln_match_symbols = match_symbols_against_resource( - vuln_patch_metadata, + vuln_evidence = match_symbols_against_resource( + patch_symbols["vulnerable"], resource_index, ) - - fixed_match_symbols = match_symbols_against_resource( - fixed_patch_metadata, + fixed_evidence = match_symbols_against_resource( + patch_symbols["fixed"], resource_index, ) - if not vuln_match_symbols and not fixed_match_symbols: + if not vuln_evidence and not fixed_evidence: continue result = { - "reachability_status": classify_reachability(vuln_match_symbols), - "summary": { - "call_paths": { - qualified_name: ev.get("reachable_from", []) - for qualified_name, ev in vuln_match_symbols.items() - if ev.get("called") - }, - }, - "evidence": vuln_match_symbols, - "vulnerable_symbols": sorted(vuln_match_symbols), - "fixed_symbols": sorted(fixed_match_symbols), + "reachability_status": classify_reachability(vuln_evidence).value, + "vulnerable_symbols": sorted(vuln_evidence), + "fixed_symbols": sorted(fixed_evidence), + "evidence": vuln_evidence, "patch": { "vcs_url": vcs_url, "commit_hash": commit_hash, }, } - print(result) - append_symbol_reachability_result(resource, result) + + resource.update_extra_data( + { + "symbols_reachability": result, + } + ) except Exception as e: logger( @@ -448,11 +444,8 @@ def collect_and_store_symbol_reachability_results(project, logger=None): f"{vcs_url}@{commit_hash}: {e}" ) finally: - if repo: - repo.close() - # cleanup_repo(repo_path) - + pass def build_resource_index(resource_text, language): if not is_supported_language(language) or not resource_text: @@ -489,7 +482,11 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): if not patch_symbols_metadata or not resource_index: return {} - call_graph = resource_index.get("call_graph") + call_graph = resource_index.get("call_graph") or {} + imports = call_graph.get("imports", {}) + + # Set of fully-qualified names the resource imports, e.g. "aiohttp.ClientSession" + imported_fq_names = set(imports.values()) target_qualified_names = { metadata["qualified_name"] for metadata in patch_symbols_metadata.values() @@ -501,24 +498,37 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): ) called_qualified_names = set() - - if call_graph: - for callees in call_graph.get("edges_qualified", {}).values(): - called_qualified_names |= set(callees) + for callees in call_graph.get("edges_qualified", {}).values(): + called_qualified_names |= set(callees) matched = {} - for metadata in patch_symbols_metadata.values(): qualified_name = metadata["qualified_name"] fingerprint = metadata["fingerprint"] - defined = qualified_name in resource_index.get("definitions", {}) + defined = qualified_name in resource_index.get("definitions", set()) fingerprint_hit = bool( - fingerprint and fingerprint in resource_index.get("fingerprints", {}) + fingerprint and fingerprint in resource_index.get("fingerprints", set()) ) - called = qualified_name in called_qualified_names - if not (defined or fingerprint_hit or called): + # Does the resource *import* this symbol? + # Match either the bare name (import key) or any fq import target + # that ends with ".". + imported = ( + qualified_name in imports + or qualified_name in imported_fq_names + or any( + fq == qualified_name or fq.endswith("." + qualified_name) + for fq in imported_fq_names + ) + ) + + called = any( + fq_name == qualified_name or fq_name.endswith("." + qualified_name) + for fq_name in called_qualified_names + ) + + if not (defined or fingerprint_hit or called or imported): continue entry = matched.setdefault( @@ -526,18 +536,29 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): { "defined": False, "called": False, + "imported": False, + "fingerprint": None, "reachable_from": [], + "external": False, }, ) - entry["defined"] = entry["defined"] or defined - entry["called"] = entry["called"] or called + if defined: + entry["defined"] = True - if fingerprint_hit: - entry["exact_match_fingerprint"] = fingerprint + if imported: + entry["imported"] = True + if not defined: + entry["external"] = True if called: + entry["called"] = True entry["reachable_from"] = sorted(reachable_callers) + if not defined: + entry["external"] = True + + if fingerprint_hit: + entry["fingerprint"] = fingerprint return matched @@ -553,8 +574,9 @@ def classify_reachability(evidence): has_path = bool(item.get("reachable_from")) is_exact = "exact_match_fingerprint" in item is_defined = bool(item.get("defined")) + is_imported = bool(item.get("imported")) - if is_called or has_path: + if is_called or has_path or is_imported: return ReachabilityStatus.REACHABLE if is_exact or is_defined: @@ -601,6 +623,7 @@ def build_call_graph(tree, language): return None index = collect_definitions(tree.root_node, language) + import_map = collect_imports(tree.root_node, language) graph_meta = {} for definition in index.values(): @@ -611,13 +634,12 @@ def build_call_graph(tree, language): continue body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_exact_symbol_fingerprint(body_text) or {} - + fingerprint = create_exact_symbol_fingerprint(body_text) graph_meta[qualified_name] = { "qualified_name": qualified_name, "node": node, "node_type": node.type, - "fingerprint": fingerprints, + "fingerprint": fingerprint, } definitions_by_name = {} @@ -635,7 +657,6 @@ def build_call_graph(tree, language): direct_calls = extract_direct_calls(metadata["node"], language, index) resolved_callees = set() - for receiver_name, callee_name in direct_calls: resolved_callees |= resolve_callee( receiver_name=receiver_name, @@ -643,6 +664,7 @@ def build_call_graph(tree, language): owner_qn=qualified_name, definitions_by_name=definitions_by_name, class_methods=class_methods, + import_map=import_map, ) edges_qualified[qualified_name] = resolved_callees @@ -650,6 +672,7 @@ def build_call_graph(tree, language): return { "nodes": graph_meta, "edges_qualified": edges_qualified, + "imports": import_map, } @@ -741,27 +764,30 @@ def get_call_receiver(callee_node): def resolve_callee( - receiver_name, callee_name, owner_qn, definitions_by_name, class_methods + receiver_name, + callee_name, + owner_qn, + definitions_by_name, + class_methods, + import_map=None, ): - """ - Resolve a call to candidate qualified names. + import_map = import_map or {} - Examples: - self.foo() from class A -> {"A.foo"} if A.foo exists - foo() -> definitions named "foo" - - """ if receiver_name == "self": owner_class = get_owner_class_name(owner_qn) - if owner_class: method_qn = f"{owner_class}.{callee_name}" - if method_qn in class_methods: return {method_qn} - candidates = definitions_by_name.get(callee_name, set()) - return set(candidates) + if callee_name in import_map: + return {import_map[callee_name]} + + if receiver_name is not None and receiver_name in import_map: + base = import_map[receiver_name] + return {f"{base}.{callee_name}"} + + return set(definitions_by_name.get(callee_name, set())) def get_owner_class_name(owner_qn): @@ -809,3 +835,46 @@ def compute_reachable_symbols(call_graph, target_qualified_names): frontier.append(parent) return reachable, bool(direct) + + +def collect_imports(root_node, language: str): + """ + Returns a dict mapping local names/aliases to their absolute import path. + Examples: + 'from django.db import models' -> {'models': 'django.db.models'} + 'import os.path' -> {'os.path': 'os.path'} + 'import numpy as np' -> {'np': 'numpy'} + 'from a.b import c as d' -> {'d': 'a.b.c'} + """ + import_map = {} + query = get_query(language, "imports") + if not query or not root_node: + return import_map + + for _pattern_index, captures in query.matches(root_node): + module_name = None + import_name = None + alias = None + + for node_name, nodes in captures.items(): + if not nodes: + continue + + text = nodes[0].text.decode("utf-8", errors="replace") + if node_name == "module_name": + module_name = text + elif node_name == "import_name": + import_name = text + elif node_name == "alias": + alias = text + + if not import_name: + continue + + local_name = alias or import_name + if module_name: + import_map[local_name] = f"{module_name}.{import_name}" + else: + import_map[local_name] = import_name + + return import_map diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index da6f359f65..6637bbf1c2 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -206,6 +206,21 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): object: (_) @receiver attribute: (identifier) @callee)) """, + "imports": """ + (import_statement name: (dotted_name) @import_name) + (import_statement + name: (aliased_import + name: (dotted_name) @import_name + alias: (identifier) @alias)) + (import_from_statement + module_name: (dotted_name) @module_name + name: (dotted_name) @import_name) + (import_from_statement + module_name: (dotted_name) @module_name + name: (aliased_import + name: (dotted_name) @import_name + alias: (identifier) @alias)) + """, }, } diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 68bbe9b964..3d5f27ab28 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -27,7 +27,7 @@ from scanpipe.models import Project from scanpipe.pipes import collect_and_create_codebase_resources -from scanpipe.pipes.reachability import ReachabilityStatus +from scanpipe.pipes.reachability import ReachabilityStatus, collect_imports, extract_direct_calls from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability @@ -485,3 +485,40 @@ def test_compute_reachable_symbols(self): expected_reachable = {"app.main", "app.helper", "app.direct_caller"} self.assertEqual(reachable, expected_reachable) + + def test_collect_imports(self): + source_code = """ +from django.db import models +import os.path +import numpy as np +from a.b import c as d + """.strip() + + tree, _ = parse_code_to_ast(source_code, "Python") + real_root_node = tree.root_node + result = collect_imports(real_root_node, language="Python") + + expected_map = { + "models": "django.db.models", + "os.path": "os.path", + "np": "numpy", + "d": "a.b.c", + } + self.assertEqual(result, expected_map) + + def test_extract_direct(self): + source_code = """ +def hello(): + return 10 + +def clean_function(): + x = 10 + y = 20 + return hello() + x + y + """.strip() + + tree, _ = parse_code_to_ast(source_code, "Python") + functions = extract_definitions(tree, "Python", kinds=("functions",)) + + result = extract_direct_calls(functions[1], "Python", []) + self.assertEqual(result, [(None, 'hello')]) \ No newline at end of file From 00fd333a6ca3ba7e52961fd2aeda68e991cd6828 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 16 Jun 2026 17:56:41 +0300 Subject: [PATCH 05/50] Add an unidiff dependency to pyproject.toml file Fix the test Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 43 ++--- .../tests/pipes/test_symbols_reachability.py | 160 ++++++++---------- 2 files changed, 91 insertions(+), 112 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index ad8a347e8f..26a1da1d8d 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -55,14 +55,14 @@ class ReachabilityStatus(str, Enum): def api_mocker(): """TODO: Remove this once the API patch url is done""" return [ - # { - # "vcs_url": "https://github.com/pallets/flask", - # "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", - # }, { - "vcs_url": "https://github.com/aio-libs/aiohttp", - "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", - } + "vcs_url": "https://github.com/pallets/flask", + "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", + }, + # { + # "vcs_url": "https://github.com/aio-libs/aiohttp", + # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", + # } ] @@ -245,6 +245,7 @@ def get_changed_lines(diff_text, file_path): return removed, added + def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -382,10 +383,8 @@ def collect_and_store_symbol_reachability_results(project, logger=None): vcs_url = patch["vcs_url"] commit_hash = patch["commit_hash"] try: - # repo_path = clone_repo(vcs_url, commit_hash) - # repo = Repo("/home/ziad-hany/PycharmProjects/flask/") - repo = Repo("/home/ziad-hany/PycharmProjects/aiohttp") - + repo_path = clone_repo(vcs_url, commit_hash) + repo = Repo(repo_path) patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) if not patch_symbols_by_language: @@ -413,6 +412,7 @@ def collect_and_store_symbol_reachability_results(project, logger=None): patch_symbols["vulnerable"], resource_index, ) + fixed_evidence = match_symbols_against_resource( patch_symbols["fixed"], resource_index, @@ -447,6 +447,7 @@ def collect_and_store_symbol_reachability_results(project, logger=None): # cleanup_repo(repo_path) pass + def build_resource_index(resource_text, language): if not is_supported_language(language) or not resource_text: return None @@ -484,8 +485,6 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): call_graph = resource_index.get("call_graph") or {} imports = call_graph.get("imports", {}) - - # Set of fully-qualified names the resource imports, e.g. "aiohttp.ClientSession" imported_fq_names = set(imports.values()) target_qualified_names = { @@ -511,9 +510,6 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): fingerprint and fingerprint in resource_index.get("fingerprints", set()) ) - # Does the resource *import* this symbol? - # Match either the bare name (import key) or any fq import target - # that ends with ".". imported = ( qualified_name in imports or qualified_name in imported_fq_names @@ -539,7 +535,6 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): "imported": False, "fingerprint": None, "reachable_from": [], - "external": False, }, ) @@ -548,14 +543,10 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): if imported: entry["imported"] = True - if not defined: - entry["external"] = True if called: entry["called"] = True entry["reachable_from"] = sorted(reachable_callers) - if not defined: - entry["external"] = True if fingerprint_hit: entry["fingerprint"] = fingerprint @@ -572,14 +563,14 @@ def classify_reachability(evidence): for item in evidence.values(): is_called = bool(item.get("called")) has_path = bool(item.get("reachable_from")) - is_exact = "exact_match_fingerprint" in item is_defined = bool(item.get("defined")) is_imported = bool(item.get("imported")) + is_exact = bool(item.get("fingerprint")) - if is_called or has_path or is_imported: + if is_exact or (is_imported and (is_called or has_path)): return ReachabilityStatus.REACHABLE - if is_exact or is_defined: + if (is_imported or is_defined) and not is_exact: highest_status = ReachabilityStatus.POTENTIALLY_REACHABLE return highest_status @@ -839,12 +830,14 @@ def compute_reachable_symbols(call_graph, target_qualified_names): def collect_imports(root_node, language: str): """ - Returns a dict mapping local names/aliases to their absolute import path. + Return a dict mapping local names/aliases to their absolute import path. + Examples: 'from django.db import models' -> {'models': 'django.db.models'} 'import os.path' -> {'os.path': 'os.path'} 'import numpy as np' -> {'np': 'numpy'} 'from a.b import c as d' -> {'d': 'a.b.c'} + """ import_map = {} query = get_query(language, "imports") diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 3d5f27ab28..1c633c3cc0 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -27,13 +27,15 @@ from scanpipe.models import Project from scanpipe.pipes import collect_and_create_codebase_resources -from scanpipe.pipes.reachability import ReachabilityStatus, collect_imports, extract_direct_calls +from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import collect_imports from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols +from scanpipe.pipes.reachability import extract_direct_calls from scanpipe.pipes.reachability import get_changed_lines from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions @@ -104,28 +106,25 @@ def test_collect_and_store_symbol_reachability_results( self.assertEqual( results, - [ - { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "summary": {"call_paths": {}}, - "evidence": { - "serve_report": { - "called": False, - "defined": True, - "reachable_from": [], - "exact_match_fingerprint": ( - "e341b914f9823915e0685396a730d421ec9e3635" - ), - } - }, - "fixed_symbols": ["serve_report"], - "vulnerable_symbols": ["serve_report"], - "reachability_status": "POTENTIALLY_REACHABLE", - } - ], + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "evidence": { + "serve_report": { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", + "reachable_from": [], + } + }, + "fixed_symbols": ["serve_report"], + "vulnerable_symbols": ["serve_report"], + "reachability_status": "REACHABLE", + }, ) def test_extract_definitions(self): @@ -210,35 +209,20 @@ def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) self.assertEqual( - classify_reachability( - {"sym1": {"exact_match_fingerprint": "hash123", "called": True}} - ), + classify_reachability({"evidence": {"fingerprint": "hash123"}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability( - { - "sym1": { - "called": True, - "reachable_from": ["main_function", "api_handler"], - } - } - ), + classify_reachability({"evidence": {"imported": True, "called": True}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"sym1": {"defined": True, "called": False}}), - ReachabilityStatus.POTENTIALLY_REACHABLE, - ) - self.assertEqual( - classify_reachability( - {"sym1": {"exact_match_fingerprint": "hash123", "called": False}} - ), + classify_reachability({"evidence": {"imported": True, "called": False}}), ReachabilityStatus.POTENTIALLY_REACHABLE, ) self.assertEqual( - classify_reachability({"sym1": {"file_path": "src/vulnerable.py"}}), + classify_reachability({"evidence": {"imported": False, "called": False}}), ReachabilityStatus.NOT_REACHABLE, ) @@ -275,7 +259,8 @@ def process_data(payload): " def inner_helper():\n" " return True\n" " return payload.strip()", - "fingerprint": "0000000888014a04b037189a42b238a2c50f218c", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" + "a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", @@ -283,7 +268,8 @@ def process_data(payload): "process_data": { "qualified_name": "process_data", "text": "def process_data(payload):\n return payload", - "fingerprint": "000000022020300e882a900807880d0300010000", + "fingerprint": "9b2797712c9ab60ea8452a441396" + "5c94d1b2f63739cab7de695e7b1dc0cf439a", "start_line": 9, "end_line": 10, "node_type": "function_definition", @@ -373,26 +359,24 @@ def test_analyze_patched_file(self): { "serve_report": { "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n " - ' """Top-level function handling a request."""\n ' - ' generator = ReportGenerator("/var/reports")\n ' - ' requested_file = request_payload.get("file")\n\n ' - "# Helper function nested inside serve_report\n " - "def build_file_path(filename):\n " - " # VULNERABLE: Direct concatenation allows Path Traversal\n " - ' # An attacker passing "../../etc/passwd" ' - "could read system files.\n " - " return os.path.join(generator.base_dir, filename)\n\n " - " if not requested_file:\n " - ' return "Error: No file specified"\n\n ' - " target_path = build_file_path(requested_file)\n\n" - " " - " " - "if os.path.exists(target_path):\n" - " " - ' return f"Serving content of {target_path}"\n\n ' - ' return "Error: File not found"', - "fingerprint": "000000556d322a47595af353274b000aa324e014", + "text": "def serve_report(request_payload):\n" + ' """Top-level function handling a request."""\n' + ' generator = ReportGenerator("/var/reports")\n' + ' requested_file = request_payload.get("file")\n\n' + " # Helper function nested inside serve_report\n" + " def build_file_path(filename):\n" + " # VULNERABLE: Direct concatenation allows Path Traversal\n" + ' # An attacker passing "../../etc/passwd"' + " could read system files.\n" + " return os.path.join(generator.base_dir, filename)\n\n" + " if not requested_file:\n" + ' return "Error: No file specified"\n\n' + " target_path = build_file_path(requested_file)\n\n" + " if os.path.exists(target_path):\n" + ' return f"Serving content of {target_path}"\n\n' + ' return "Error: File not found"', + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", "start_line": 11, "end_line": 30, "node_type": "function_definition", @@ -405,27 +389,30 @@ def test_analyze_patched_file(self): { "serve_report": { "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n " - ' """Top-level function handling a request."""\n ' - ' generator = ReportGenerator("/var/reports")\n ' - ' requested_file = request_payload.get("file")\n\n ' - " # Helper function nested inside serve_report\n " - " def build_file_path(filename):\n " - " # FIXED: Validate that the resolved " - "path stays within the base_dir\n " - " base = os.path.abspath(generator.base_dir)\n " - " target = os.path.abspath(os.path.join(base, filename))\n " - " if not target.startswith(base):\n " - ' raise ValueError("Path Traversal Detected")\n ' - " return target\n\n if not requested_file:\n " - ' return "Error: No file specified"\n\n try:\n ' - " target_path = build_file_path(requested_file)\n " - " except ValueError:\n " - ' return "Error: Invalid path"\n\n' - " if os.path.exists(target_path):\n " - ' return f"Serving content of {target_path}"\n\n ' - ' return "Error: File not found"', - "fingerprint": "0000006cceea8aedf1da91830f67b64927086d24", + "text": "def serve_report(request_payload):\n" + ' """Top-level function handling a request."""\n' + ' generator = ReportGenerator("/var/reports")\n' + ' requested_file = request_payload.get("file")\n\n' + " # Helper function nested inside serve_report\n" + " def build_file_path(filename):\n" + " # FIXED: Validate that the resolved" + " path stays within the base_dir\n" + " base = os.path.abspath(generator.base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n" + ' raise ValueError("Path Traversal Detected")\n' + " return target\n\n" + " if not requested_file:\n " + ' return "Error: No file specified"\n\n' + " try:\n" + " target_path = build_file_path(requested_file)\n" + " except ValueError:\n" + ' return "Error: Invalid path"\n\n ' + " if os.path.exists(target_path):\n" + ' return f"Serving content of {target_path}"\n\n' + ' return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d7" + "3d9afdfc3f469ccf86e8835915d240e76", "start_line": 11, "end_line": 36, "node_type": "function_definition", @@ -510,7 +497,6 @@ def test_extract_direct(self): source_code = """ def hello(): return 10 - def clean_function(): x = 10 y = 20 @@ -521,4 +507,4 @@ def clean_function(): functions = extract_definitions(tree, "Python", kinds=("functions",)) result = extract_direct_calls(functions[1], "Python", []) - self.assertEqual(result, [(None, 'hello')]) \ No newline at end of file + self.assertEqual(result, [(None, "hello")]) From 5da2504d7265b67a0bc475ca0c66a12e03a32bb3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 16 Jun 2026 18:53:55 +0300 Subject: [PATCH 06/50] Add unidiff to package dependencies to parse diff text Signed-off-by: ziad hany --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 4b3f331a1e..50a4286150 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ dependencies = [ "urllib3==2.7.0", "idna==3.18", "GitPython==3.1.50", + "unidiff==0.7.5", "lxml==6.1.1", "certifi==2026.6.17", # Profiling From 1e7f76e921605a600d8c269b2bcb47eaf894057f Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 24 Jun 2026 02:03:15 +0300 Subject: [PATCH 07/50] Simplify the pipeline logic for imports and direct calls Add missing logic for imports Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 511 ++++++++++-------- scanpipe/pipes/symbols.py | 77 +-- .../tests/pipes/test_symbols_reachability.py | 63 ++- 3 files changed, 357 insertions(+), 294 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 26a1da1d8d..1e52eadeff 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -23,8 +23,11 @@ import os import shutil import tempfile +from dataclasses import dataclass +from dataclasses import field from enum import Enum from pathlib import Path +from typing import Any from git import Repo from git.diff import NULL_TREE @@ -35,11 +38,10 @@ from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import _root_of from scanpipe.pipes.symbols import collect_definitions -from scanpipe.pipes.symbols import create_exact_symbol_fingerprint +from scanpipe.pipes.symbols import create_sha256_fingerprint from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols from scanpipe.pipes.symbols import get_query -from scanpipe.pipes.symbols import is_nested_function from scanpipe.pipes.symbols import parse_code_to_ast from scanpipe.pipes.symbols import qualified_name_from_index @@ -330,7 +332,7 @@ def collect_patch_symbols(repo, commit_hash): } """ - commit, parent = get_commit_and_parent(repo, commit_hash) + commit, parent = get_commit_and_parent(repo=repo, commit_hash=commit_hash) diff_text = get_commit_diff_text(repo, parent, commit) changed = get_changed_files(parent, commit) @@ -383,8 +385,11 @@ def collect_and_store_symbol_reachability_results(project, logger=None): vcs_url = patch["vcs_url"] commit_hash = patch["commit_hash"] try: - repo_path = clone_repo(vcs_url, commit_hash) - repo = Repo(repo_path) + # repo_path = clone_repo(vcs_url, commit_hash) + # repo = Repo(repo_path) + + repo = Repo("/home/ziad-hany/PycharmProjects/flask") + # repo = Repo("/home/ziad-hany/PycharmProjects/vulnerablecode") patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) if not patch_symbols_by_language: @@ -422,14 +427,18 @@ def collect_and_store_symbol_reachability_results(project, logger=None): continue result = { - "reachability_status": classify_reachability(vuln_evidence).value, - "vulnerable_symbols": sorted(vuln_evidence), - "fixed_symbols": sorted(fixed_evidence), - "evidence": vuln_evidence, - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, + "symbols_reachability": { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability( + vuln_evidence + ).value, + } } resource.update_extra_data( @@ -448,43 +457,147 @@ def collect_and_store_symbol_reachability_results(project, logger=None): pass -def build_resource_index(resource_text, language): +@dataclass +class SymbolNode: + qualified_name: str + node: Any # tree-sitter node + node_type: str + fingerprint: str + + +@dataclass +class CallGraph: + nodes: dict[str, SymbolNode] = field(default_factory=dict) + edges_qualified: dict[str, set[str]] = field(default_factory=dict) + imports: dict[str, str] = field(default_factory=dict) + + +@dataclass +class ResourceIndex: + definitions: set[str] = field(default_factory=set) + fingerprints: set[str] = field(default_factory=set) + call_graph: CallGraph | None = None + + def to_dict(self): + return { + "definitions": self.definitions, + "fingerprints": self.fingerprints, + "call_graph": { + "nodes": {k: vars(v) for k, v in self.call_graph.nodes.items()}, + "edges_qualified": self.call_graph.edges_qualified, + "imports": self.call_graph.imports, + } + if self.call_graph + else None, + } + + +class CallGraphBuilder: + def __init__(self, tree, language: str): + self.tree = tree + self.language = language + self.index = collect_definitions(tree.root_node, language) + self.imports = collect_imports(tree.root_node, language) + + self.nodes: dict[str, SymbolNode] = {} + self.definitions_by_name: dict[str, set[str]] = {} + self.class_methods: set[str] = set() + + def build(self) -> CallGraph: + """Main execution flow for building the call graph.""" + self._extract_nodes() + edges = self._resolve_all_edges() + + return CallGraph(nodes=self.nodes, edges_qualified=edges, imports=self.imports) + + def _extract_nodes(self): + """Extract all definitions and populate lookup maps.""" + sep = get_imports_language_config(self.language)["separator"] + + for definition in self.index.values(): + node = definition["node"] + qualified_name = qualified_name_from_index(node, self.index) + + if not qualified_name: + continue + + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + + self.nodes[qualified_name] = SymbolNode( + qualified_name=qualified_name, + node=node, + node_type=node.type, + fingerprint=fingerprint, + ) + + short_name = qualified_name.rsplit(sep, 1)[-1] + self.definitions_by_name.setdefault(short_name, set()).add(qualified_name) + + if node.type == "function_definition": + parent = node.parent + if parent is not None and parent.type == "class_definition": + self.class_methods.add(qualified_name) + + def _resolve_all_edges(self) -> dict[str, set[str]]: + """Map out all direct calls for every node in the graph.""" + edges_qualified = {} + + for qualified_name, symbol_node in self.nodes.items(): + direct_calls = extract_direct_calls(symbol_node.node, self.language) + resolved_callees = set() + + for receiver_name, callee_name in direct_calls: + resolved_callees |= resolve_callee( + receiver_name=receiver_name, + callee_name=callee_name, + owner_qn=qualified_name, + definitions_by_name=self.definitions_by_name, + class_methods=self.class_methods, + import_map=self.imports, + language=self.language, + ) + + edges_qualified[qualified_name] = resolved_callees + + return edges_qualified + + +def build_resource_index(resource_text, language) -> ResourceIndex | None: if not is_supported_language(language) or not resource_text: return None tree, _ = parse_code_to_ast(resource_text, language) - if tree is None: return None call_graph = build_call_graph(tree, language) - meta = ( - call_graph["nodes"] - if call_graph - else build_symbol_metadata( - extract_definitions(tree, language), - language, - ) - ) + if call_graph: + definitions = set(call_graph.nodes.keys()) + fingerprints = { + node.fingerprint for node in call_graph.nodes.values() if node.fingerprint + } + else: + meta = build_symbol_metadata(extract_definitions(tree, language), language) + definitions = {m["qualified_name"] for m in meta.values()} + fingerprints = {m["fingerprint"] for m in meta.values() if m["fingerprint"]} - return { - "definitions": {metadata["qualified_name"] for metadata in meta.values()}, - "fingerprints": { - metadata["fingerprint"] - for metadata in meta.values() - if metadata["fingerprint"] - }, - "call_graph": call_graph, - } + return ResourceIndex( + definitions=definitions, fingerprints=fingerprints, call_graph=call_graph + ) -def match_symbols_against_resource(patch_symbols_metadata, resource_index): +def match_symbols_against_resource( + patch_symbols_metadata: dict[str, Any], resource_index: "ResourceIndex" +) -> dict[str, Any]: if not patch_symbols_metadata or not resource_index: return {} - call_graph = resource_index.get("call_graph") or {} - imports = call_graph.get("imports", {}) + call_graph = resource_index.call_graph + imports = call_graph.imports if call_graph else {} + edges_qualified = call_graph.edges_qualified if call_graph else {} + imported_fq_names = set(imports.values()) target_qualified_names = { @@ -492,12 +605,12 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): } reachable_callers, _ = compute_reachable_symbols( - call_graph, + edges_qualified, target_qualified_names, ) called_qualified_names = set() - for callees in call_graph.get("edges_qualified", {}).values(): + for callees in edges_qualified.values(): called_qualified_names |= set(callees) matched = {} @@ -505,9 +618,9 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): qualified_name = metadata["qualified_name"] fingerprint = metadata["fingerprint"] - defined = qualified_name in resource_index.get("definitions", set()) + defined = qualified_name in resource_index.definitions fingerprint_hit = bool( - fingerprint and fingerprint in resource_index.get("fingerprints", set()) + fingerprint and fingerprint in resource_index.fingerprints ) imported = ( @@ -530,8 +643,9 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): entry = matched.setdefault( qualified_name, { - "defined": False, + "symbol_name": qualified_name, "called": False, + "defined": False, "imported": False, "fingerprint": None, "reachable_from": [], @@ -540,15 +654,12 @@ def match_symbols_against_resource(patch_symbols_metadata, resource_index): if defined: entry["defined"] = True - if imported: entry["imported"] = True - if called: entry["called"] = True entry["reachable_from"] = sorted(reachable_callers) - - if fingerprint_hit: + if fingerprint: entry["fingerprint"] = fingerprint return matched @@ -558,8 +669,7 @@ def classify_reachability(evidence): if not evidence: return ReachabilityStatus.NOT_REACHABLE - highest_status = ReachabilityStatus.NOT_REACHABLE - + status = ReachabilityStatus.NOT_REACHABLE for item in evidence.values(): is_called = bool(item.get("called")) has_path = bool(item.get("reachable_from")) @@ -571,9 +681,9 @@ def classify_reachability(evidence): return ReachabilityStatus.REACHABLE if (is_imported or is_defined) and not is_exact: - highest_status = ReachabilityStatus.POTENTIALLY_REACHABLE + status = ReachabilityStatus.POTENTIALLY_REACHABLE - return highest_status + return status def build_symbol_metadata(nodes, language, index=None): @@ -582,15 +692,12 @@ def build_symbol_metadata(nodes, language, index=None): metadata = {} for node in nodes: - if is_nested_function(node, language): - continue - qualified_name = qualified_name_from_index(node, index) if not qualified_name: continue body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_exact_symbol_fingerprint(body_text) + fingerprints = create_sha256_fingerprint(body_text) key = qualified_name suffix = 1 @@ -609,65 +716,15 @@ def build_symbol_metadata(nodes, language, index=None): return metadata -def build_call_graph(tree, language): +def build_call_graph(tree, language) -> CallGraph | None: if tree is None or not is_supported_language(language): return None - index = collect_definitions(tree.root_node, language) - import_map = collect_imports(tree.root_node, language) + builder = CallGraphBuilder(tree, language) + return builder.build() - graph_meta = {} - for definition in index.values(): - node = definition["node"] - qualified_name = qualified_name_from_index(node, index) - - if not qualified_name: - continue - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_exact_symbol_fingerprint(body_text) - graph_meta[qualified_name] = { - "qualified_name": qualified_name, - "node": node, - "node_type": node.type, - "fingerprint": fingerprint, - } - - definitions_by_name = {} - class_methods = set() - - for qualified_name, metadata in graph_meta.items(): - name = qualified_name.rsplit(".", 1)[-1] - definitions_by_name.setdefault(name, set()).add(qualified_name) - - if metadata["node_type"] == "function_definition" and "." in qualified_name: - class_methods.add(qualified_name) - - edges_qualified = {} - for qualified_name, metadata in graph_meta.items(): - direct_calls = extract_direct_calls(metadata["node"], language, index) - - resolved_callees = set() - for receiver_name, callee_name in direct_calls: - resolved_callees |= resolve_callee( - receiver_name=receiver_name, - callee_name=callee_name, - owner_qn=qualified_name, - definitions_by_name=definitions_by_name, - class_methods=class_methods, - import_map=import_map, - ) - - edges_qualified[qualified_name] = resolved_callees - - return { - "nodes": graph_meta, - "edges_qualified": edges_qualified, - "imports": import_map, - } - - -def extract_direct_calls(node, language, definition_index): +def extract_direct_calls(node, language): """ Return direct calls inside `node`, excluding calls inside nested definitions. @@ -684,54 +741,30 @@ def extract_direct_calls(node, language, definition_index): if query is None or node is None: return [] - definition_ids = set(definition_index) calls = [] + definition_types = {"function_definition", "class_definition"} for _, captures in query.matches(node): for callee_node in captures.get("callee", []): - if is_inside_nested_definition( - node=callee_node, - owner_node=node, - definition_ids=definition_ids, - ): + cur = callee_node.parent + inside_nested = False + while cur is not None and cur.id != node.id: + if cur.type in definition_types: + inside_nested = True + break + cur = cur.parent + + if inside_nested: continue receiver_name = get_call_receiver(callee_node) - callee_name = node_text(callee_node) + callee_name = callee_node.text.decode("utf-8", errors="replace") if callee_name: calls.append((receiver_name, callee_name)) - return calls -def is_inside_nested_definition(node, owner_node, definition_ids): - """ - Return True if node is inside a nested function/class within `owner_node`. - - Example: - def outer(): - foo() # belongs to outer - - def inner(): - bar() # nested; should not count as outer's call - - """ - current = node.parent - - while current is not None and current is not owner_node: - if current.id in definition_ids: - return True - - current = current.parent - - return False - - -def node_text(node): - return node.text.decode("utf-8", errors="replace") - - def get_call_receiver(callee_node): """ Return receiver name for attribute calls. @@ -751,50 +784,7 @@ def get_call_receiver(callee_node): if object_node is None: return None - return node_text(object_node) - - -def resolve_callee( - receiver_name, - callee_name, - owner_qn, - definitions_by_name, - class_methods, - import_map=None, -): - import_map = import_map or {} - - if receiver_name == "self": - owner_class = get_owner_class_name(owner_qn) - if owner_class: - method_qn = f"{owner_class}.{callee_name}" - if method_qn in class_methods: - return {method_qn} - - if callee_name in import_map: - return {import_map[callee_name]} - - if receiver_name is not None and receiver_name in import_map: - base = import_map[receiver_name] - return {f"{base}.{callee_name}"} - - return set(definitions_by_name.get(callee_name, set())) - - -def get_owner_class_name(owner_qn): - """ - Return enclosing class name from a qualified name. - - Examples: - "User.save" -> "User" - "User.Inner.save" -> "User.Inner" - "save" -> None - - """ - if "." not in owner_qn: - return None - - return owner_qn.rsplit(".", 1)[0] + return object_node.text.decode("utf-8", errors="replace") def compute_reachable_symbols(call_graph, target_qualified_names): @@ -828,46 +818,141 @@ def compute_reachable_symbols(call_graph, target_qualified_names): return reachable, bool(direct) -def collect_imports(root_node, language: str): - """ - Return a dict mapping local names/aliases to their absolute import path. +def get_imports_language_config(language: str) -> dict: + configs = { + "Python": {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"}, + "Javascript": { + "self_keyword": "this", + "separator": ".", + "wildcard_symbol": "*", + }, + "Java": {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"}, + "C++": {"self_keyword": "this", "separator": "::", "wildcard_symbol": None}, + "PHP": {"self_keyword": "$this", "separator": "::", "wildcard_symbol": None}, + "Go": {"self_keyword": None, "separator": "/", "wildcard_symbol": None}, + "Ruby": {"self_keyword": "self", "separator": "::", "wildcard_symbol": None}, + } + return configs.get( + language, {"self_keyword": None, "separator": ".", "wildcard_symbol": None} + ) - Examples: - 'from django.db import models' -> {'models': 'django.db.models'} - 'import os.path' -> {'os.path': 'os.path'} - 'import numpy as np' -> {'np': 'numpy'} - 'from a.b import c as d' -> {'d': 'a.b.c'} +def resolve_callee( + receiver_name: str | None, + callee_name: str, + owner_qn: str, + definitions_by_name: dict[str, set[str]], + class_methods: set[str], + language: str, + import_map: dict | None = None, +) -> set[str]: + config = get_imports_language_config(language) + sep = config["separator"] + self_kw = config["self_keyword"] + import_map = import_map or {} + + if receiver_name: + receiver_name = receiver_name.strip("'\"") + if callee_name: + callee_name = callee_name.strip("'\"") + + if self_kw is not None and receiver_name == self_kw: + owner_class = owner_qn.rsplit(sep, 1)[0] if sep in owner_qn else owner_qn + return {f"{owner_class}{sep}{callee_name}"} + + if callee_name in import_map: + return {import_map[callee_name]} + + if receiver_name is not None and receiver_name != self_kw: + candidates = set() + + if receiver_name in import_map and receiver_name != "*": + base = import_map[receiver_name] + candidates.add(f"{base}{sep}{callee_name}") + + static_fqn = f"{receiver_name}{sep}{callee_name}" + if static_fqn in class_methods: + candidates.add(static_fqn) + + return candidates + + local_defs = definitions_by_name.get(callee_name, set()) + if local_defs: + return local_defs + + if "*" in import_map: + return {f"{mod}{sep}{callee_name}" for mod in import_map["*"]} + + return set() + + +def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: + """ + Extract import statements from a Tree-sitter AST and map every local alias to its absolute imported path. """ - import_map = {} + config = get_imports_language_config(language) + separator = config["separator"] + wildcard_sym = config.get("wildcard_symbol") + query = get_query(language, "imports") if not query or not root_node: - return import_map + return {} + + import_map: dict[str, str | list[str]] = {} + wildcard_modules: list[str] = [] for _pattern_index, captures in query.matches(root_node): + flat_captures = [] + for tag, nodes in captures.items(): + for n in nodes: + flat_captures.append( + ( + n.start_byte, + tag, + n.text.decode("utf-8", errors="replace").strip("'\""), + ) + ) + + flat_captures.sort(key=lambda x: x[0]) + module_name = None - import_name = None - alias = None + current_import = None + pairs = [] - for node_name, nodes in captures.items(): - if not nodes: + for _, tag, text in flat_captures: + if tag == "module_name": + module_name = text + elif tag == "import_name": + if current_import is not None: + pairs.append((current_import, None)) + current_import = text + elif tag == "alias": + pairs.append((current_import, text)) + current_import = None + + if current_import is not None: + pairs.append((current_import, None)) + + for imp_name, alias in pairs: + if not imp_name: continue - text = nodes[0].text.decode("utf-8", errors="replace") - if node_name == "module_name": - module_name = text - elif node_name == "import_name": - import_name = text - elif node_name == "alias": - alias = text + if wildcard_sym is not None and imp_name == wildcard_sym: + if module_name: + wildcard_modules.append(module_name) + continue - if not import_name: - continue + local_name = alias or imp_name + + if not alias and separator in imp_name: + local_name = imp_name.split(separator)[0] + + absolute_path = ( + f"{module_name}{separator}{imp_name}" if module_name else imp_name + ) + import_map[local_name] = absolute_path - local_name = alias or import_name - if module_name: - import_map[local_name] = f"{module_name}.{import_name}" - else: - import_map[local_name] = import_name + if wildcard_modules: + import_map["*"] = wildcard_modules return import_map diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 6637bbf1c2..b4dd67fb44 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -207,19 +207,23 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): attribute: (identifier) @callee)) """, "imports": """ - (import_statement name: (dotted_name) @import_name) - (import_statement - name: (aliased_import - name: (dotted_name) @import_name - alias: (identifier) @alias)) - (import_from_statement - module_name: (dotted_name) @module_name - name: (dotted_name) @import_name) + (import_statement name: [ + (dotted_name) @import_name + (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + ]) + + ; Combine explicit, relative, aliased, and wildcard from_imports (import_from_statement - module_name: (dotted_name) @module_name - name: (aliased_import + module_name: [ + (dotted_name) @module_name + (relative_import) @module_name + ] + [ name: (dotted_name) @import_name - alias: (identifier) @alias)) + name: (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + (wildcard_import) @import_name + ] + ) """, }, } @@ -274,61 +278,12 @@ def run_query(query: Query, root_node): yield def_nodes[0], name -def query_captures(language, kind, node): - """Re-run a definition query on the root of node's tree.""" - query = get_query(language, kind) - return list(run_query(query, _root_of(node))) - - def _root_of(node): while node.parent is not None: node = node.parent return node -def is_nested_function(node, language): - function_nodes = { - captured_node - for captured_node, _ in query_captures(language, "functions", node) - } - class_nodes = { - captured_node for captured_node, _ in query_captures(language, "classes", node) - } - - if node not in function_nodes: - return False - - function_types = {captured_node.type for captured_node in function_nodes} - class_types = {captured_node.type for captured_node in class_nodes} - - parent = node.parent - - while parent is not None: - if parent.type in function_types: - return True - - if parent.type in class_types: - return False - - parent = parent.parent - - return False - - -def extract_calls_in_node(node, language: str): - query = get_query(language, "calls") - if query is None or node is None: - return set() - - names = set() - for _pattern_index, captures in query.matches(node): - for callee_node in captures.get("callee", []): - name = callee_node.text.decode("utf-8", errors="replace") - if name: - names.add(name) - return names - - def collect_definitions(root_node, language: str): index: dict[int, dict] = {} for kind in ("functions", "classes"): @@ -381,7 +336,7 @@ def qualified_name_from_index(node, index): return ".".join(reversed(parts)) -def create_exact_symbol_fingerprint(text): +def create_sha256_fingerprint(text): if text is None: return None diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 1c633c3cc0..27b3dbb1e3 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -107,23 +107,36 @@ def test_collect_and_store_symbol_reachability_results( self.assertEqual( results, { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "evidence": { - "serve_report": { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", - "reachable_from": [], - } - }, - "fixed_symbols": ["serve_report"], - "vulnerable_symbols": ["serve_report"], - "reachability_status": "REACHABLE", + "symbols_reachability": { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c00679511833553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": [], + }, + ], + "fixed_symbols": ["serve_report", "serve_report.build_file_path"], + "vulnerable_symbols": [ + "serve_report", + "serve_report.build_file_path", + ], + "reachability_status": "REACHABLE", + } }, ) @@ -479,6 +492,9 @@ def test_collect_imports(self): import os.path import numpy as np from a.b import c as d +from . import utils +from ..core import engine +from math import * """.strip() tree, _ = parse_code_to_ast(source_code, "Python") @@ -487,10 +503,14 @@ def test_collect_imports(self): expected_map = { "models": "django.db.models", - "os.path": "os.path", + "os": "os.path", "np": "numpy", "d": "a.b.c", + "utils": "..utils", + "engine": "..core.engine", + "*": ["math"], } + self.assertEqual(result, expected_map) def test_extract_direct(self): @@ -506,5 +526,8 @@ def clean_function(): tree, _ = parse_code_to_ast(source_code, "Python") functions = extract_definitions(tree, "Python", kinds=("functions",)) - result = extract_direct_calls(functions[1], "Python", []) - self.assertEqual(result, [(None, "hello")]) + result = extract_direct_calls(functions[1], "Python") + self.assertEqual( + result, + [(None, "hello")], + ) From 4428760a2e7ee0f4e7526a9c244a298d40a133e1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 24 Jun 2026 13:45:42 +0300 Subject: [PATCH 08/50] Add support for vulnerablecode reachability option Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 160 +++++++++++++++---------------- scanpipe/pipes/vulnerablecode.py | 1 + 2 files changed, 76 insertions(+), 85 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 1e52eadeff..de070b373e 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -54,20 +54,6 @@ class ReachabilityStatus(str, Enum): NOT_REACHABLE = "NOT_REACHABLE" -def api_mocker(): - """TODO: Remove this once the API patch url is done""" - return [ - { - "vcs_url": "https://github.com/pallets/flask", - "commit_hash": "089cb86dd22bff589a4eafb7ab8e42dc357623b4", - }, - # { - # "vcs_url": "https://github.com/aio-libs/aiohttp", - # "commit_hash": "0c2e9da51126238a421568eb7c5b53e5b5d17b36", - # } - ] - - def clone_repo(vcs_url, commit_hash=None): repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") @@ -381,80 +367,96 @@ def collect_and_store_symbol_reachability_results(project, logger=None): is_media=False, ) - for patch in api_mocker(): - vcs_url = patch["vcs_url"] - commit_hash = patch["commit_hash"] - try: - # repo_path = clone_repo(vcs_url, commit_hash) - # repo = Repo(repo_path) + vulnerabilities = project.package_vulnerabilities + + for vulnerability in vulnerabilities: + patches = vulnerability.get("fixed_in_patches", []) - repo = Repo("/home/ziad-hany/PycharmProjects/flask") - # repo = Repo("/home/ziad-hany/PycharmProjects/vulnerablecode") - patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) + cloned_repos = {} + for patch in patches: + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") - if not patch_symbols_by_language: + if not vcs_url or not commit_hash: continue - for resource in candidate_resources: - resource_language = resource.programming_language - if resource_language not in patch_symbols_by_language: - continue + try: + if vcs_url not in cloned_repos: + cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) - resource_text = resource.file_content - if not resource_text: - continue + repo_path = cloned_repos[vcs_url] + repo = Repo(repo_path) - patch_symbols = patch_symbols_by_language[resource_language] - resource_index = build_resource_index( - resource_text, - resource_language, - ) + patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) - if not resource_index: + if not patch_symbols_by_language: continue - vuln_evidence = match_symbols_against_resource( - patch_symbols["vulnerable"], - resource_index, - ) + for resource in candidate_resources: + resource_language = resource.programming_language + if resource_language not in patch_symbols_by_language: + continue - fixed_evidence = match_symbols_against_resource( - patch_symbols["fixed"], - resource_index, - ) + resource_text = resource.file_content + if not resource_text: + continue - if not vuln_evidence and not fixed_evidence: - continue + patch_symbols = patch_symbols_by_language[resource_language] + resource_index = build_resource_index( + resource_text, + resource_language, + ) - result = { - "symbols_reachability": { - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, - "evidence": list(vuln_evidence.values()), - "fixed_symbols": sorted(fixed_evidence.keys()), - "vulnerable_symbols": sorted(vuln_evidence.keys()), - "reachability_status": classify_reachability( - vuln_evidence - ).value, - } - } + if not resource_index: + continue + + vuln_evidence = match_symbols_against_resource( + patch_symbols["vulnerable"], + resource_index, + ) - resource.update_extra_data( - { - "symbols_reachability": result, + fixed_evidence = match_symbols_against_resource( + patch_symbols["fixed"], + resource_index, + ) + + if not vuln_evidence and not fixed_evidence: + continue + + result = { + "symbols_reachability": { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability( + vuln_evidence + ).value, + } } - ) - except Exception as e: - logger( - f"Failed to collect symbol reachability for " - f"{vcs_url}@{commit_hash}: {e}" - ) - finally: - # cleanup_repo(repo_path) - pass + resource.update_extra_data( + { + "symbols_reachability": result, + } + ) + print(result) + except Exception as e: + if logger: + logger( + f"Failed to collect symbol reachability for " + f"{vcs_url}@{commit_hash}: {e}" + ) + + for url, path in cloned_repos.items(): + try: + cleanup_repo(path) + except Exception as e: + if logger: + logger(f"Failed to clean up repo {url} at {path}: {e}") @dataclass @@ -742,21 +744,9 @@ def extract_direct_calls(node, language): return [] calls = [] - definition_types = {"function_definition", "class_definition"} for _, captures in query.matches(node): for callee_node in captures.get("callee", []): - cur = callee_node.parent - inside_nested = False - while cur is not None and cur.id != node.id: - if cur.type in definition_types: - inside_nested = True - break - cur = cur.parent - - if inside_nested: - continue - receiver_name = get_call_receiver(callee_node) callee_name = callee_node.text.decode("utf-8", errors="replace") diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 20f5e69b5f..4deaefc704 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -118,6 +118,7 @@ def bulk_search_by_purl( data = { "purls": purls, "details": True, + "reachability": True, } logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}") From e1a2d8651056362db6b2eb1005b6937a4c0e13c3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 24 Jun 2026 17:58:43 +0300 Subject: [PATCH 09/50] Fix the tests and improve patch extraction performance Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 120 ++++++++-------- .../tests/pipes/test_symbols_reachability.py | 128 ++++++++++-------- 2 files changed, 130 insertions(+), 118 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index de070b373e..4a156c9b60 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -182,58 +182,6 @@ def get_changed_files(parent_commit, commit): return files -def get_changed_lines(diff_text, file_path): - """ - Return `(removed_lines, added_lines)` for one file. - - For pure-insertion hunks (no removed lines) we anchor the vulnerable side - to the hunk's source location so the enclosing old symbol is still found. - For pure-deletion hunks we do the mirror image on the added side. - """ - removed = [] - added = [] - - if not diff_text: - return removed, added - - for patched_file in PatchSet.from_string(diff_text): - candidates = { - patched_file.path, - (patched_file.source_file or "").removeprefix("a/"), - (patched_file.target_file or "").removeprefix("b/"), - } - if file_path not in candidates: - continue - - for hunk in patched_file: - hunk_removed = [ - line.source_line_no - for line in hunk - if line.is_removed and line.source_line_no - ] - hunk_added = [ - line.target_line_no - for line in hunk - if line.is_added and line.target_line_no - ] - - # Pure insertion: nothing removed -> anchor old side to the - # line just before the insertion point in the source file. - if hunk_added and not hunk_removed: - anchor = max(hunk.source_start, 1) - hunk_removed = [anchor] - - # Pure deletion: nothing added -> anchor new side similarly. - if hunk_removed and not hunk_added: - anchor = max(hunk.target_start, 1) - hunk_added = [anchor] - - removed.extend(hunk_removed) - added.extend(hunk_added) - - return removed, added - - def diff_changed_symbols(vuln_meta, fixed_meta): """ Keep only symbols whose body actually differs between vulnerable and fixed @@ -254,7 +202,9 @@ def diff_changed_symbols(vuln_meta, fixed_meta): return vuln_only, fixed_only -def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): +def analyze_patched_file( + vulnerable_text, fixed_text, removed_lines, added_lines, file_path +): """ Return `(vuln_metadata, fixed_metadata, language)` for one changed file, restricted to symbols actually touched by the patch. @@ -282,8 +232,6 @@ def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): if vuln_tree is None and fixed_tree is None: return {}, {}, language - removed_lines, added_lines = get_changed_lines(diff_text, file_path) - vuln_nodes = ( extract_symbols(vuln_tree, removed_lines, language) if vuln_tree else [] ) @@ -300,6 +248,57 @@ def analyze_patched_file(vulnerable_text, fixed_text, diff_text, file_path): return vuln_meta, fixed_meta, language +def parse_diff_lines(diff_text) -> dict[str, tuple[list[int], list[int]]]: + """ + Parses the entire unified diff text once and maps each file path to its tuple of (removed_lines, added_lines). + """ + diff_map = {} + if not diff_text: + return diff_map + + for patched_file in PatchSet.from_string(diff_text): + removed = [] + added = [] + + for hunk in patched_file: + hunk_removed = [ + line.source_line_no + for line in hunk + if line.is_removed and line.source_line_no + ] + hunk_added = [ + line.target_line_no + for line in hunk + if line.is_added and line.target_line_no + ] + + # Pure insertion anchor + if hunk_added and not hunk_removed: + anchor = max(hunk.source_start, 1) + hunk_removed = [anchor] + + # Pure deletion anchor + if hunk_removed and not hunk_added: + anchor = max(hunk.target_start, 1) + hunk_added = [anchor] + + removed.extend(hunk_removed) + added.extend(hunk_added) + + # Register the line tracking data against all naming variations of this file + candidates = { + patched_file.path, + (patched_file.source_file or "").removeprefix("a/"), + (patched_file.target_file or "").removeprefix("b/"), + } + + for candidate in candidates: + if candidate: + diff_map[candidate] = (removed, added) + + return diff_map + + def collect_patch_symbols(repo, commit_hash): """ Return: @@ -319,17 +318,21 @@ def collect_patch_symbols(repo, commit_hash): """ commit, parent = get_commit_and_parent(repo=repo, commit_hash=commit_hash) - diff_text = get_commit_diff_text(repo, parent, commit) - changed = get_changed_files(parent, commit) + diff_text = get_commit_diff_text(repo=repo, parent_commit=parent, commit=commit) + changed = get_changed_files(parent_commit=parent, commit=commit) + diff_line_map = parse_diff_lines(diff_text=diff_text) by_language = {} for file_path, texts in changed.items(): vulnerable_text = texts["vulnerable_text"] fixed_text = texts["fixed_text"] + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + vuln_meta, fixed_meta, language = analyze_patched_file( vulnerable_text=vulnerable_text, fixed_text=fixed_text, - diff_text=diff_text, + removed_lines=removed_lines, + added_lines=added_lines, file_path=file_path, ) @@ -443,7 +446,6 @@ def collect_and_store_symbol_reachability_results(project, logger=None): "symbols_reachability": result, } ) - print(result) except Exception as e: if logger: logger( diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 27b3dbb1e3..93937dc857 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -21,6 +21,7 @@ # Visit https://github.com/nexB/scancode.io for support and download. from pathlib import Path +from unittest.mock import PropertyMock from unittest.mock import patch from django.test import TestCase @@ -36,7 +37,7 @@ from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols from scanpipe.pipes.reachability import extract_direct_calls -from scanpipe.pipes.reachability import get_changed_lines +from scanpipe.pipes.reachability import parse_diff_lines from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions from scanpipe.pipes.symbols import extract_symbols @@ -51,31 +52,41 @@ def setUp(self): self.project1 = Project.objects.create(name="Analysis") self.project1.codebase_path.mkdir(parents=True, exist_ok=True) - @patch("scanpipe.pipes.reachability.Repo") @patch("scanpipe.pipes.reachability.clone_repo") - @patch("scanpipe.pipes.reachability.api_mocker") @patch("scanpipe.pipes.reachability.collect_patch_symbols") + @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_collect_and_store_symbol_reachability_results( - self, mock_collect_symbols, mock_api, mock_clone_repo, mock_repo + self, + mock_package_vulnerabilities, + mock_collect_symbols, + mock_clone_repo, ): app_text = (self.data / "app.py").read_text() vuln_text = (self.data / "vuln-app.py").read_text() fixed_text = (self.data / "fixed-app.py").read_text() diff_text = (self.data / "diff-app.patch").read_text() + diff_line_map = parse_diff_lines(diff_text=diff_text) + file_path = "app.py" + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) vuln_meta, fixed_meta, lang = analyze_patched_file( vulnerable_text=vuln_text, fixed_text=fixed_text, - diff_text=diff_text, - file_path="app.py", + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, ) self.assertTrue(lang) self.assertTrue(vuln_meta or fixed_meta) - mock_api.return_value = [ + mock_package_vulnerabilities.return_value = [ { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + "fixed_in_patches": [ + { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + } + ] } ] @@ -99,7 +110,8 @@ def test_collect_and_store_symbol_reachability_results( resource.programming_language = lang resource.save() - collect_and_store_symbol_reachability_results(self.project1) + with patch("scanpipe.pipes.reachability.Repo"): + collect_and_store_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") @@ -114,19 +126,20 @@ def test_collect_and_store_symbol_reachability_results( }, "evidence": [ { + "symbol_name": "serve_report", "called": False, "defined": True, "imported": False, "fingerprint": "d7675efb263896da2a3c00679511833553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", "reachable_from": [], }, { - "called": False, + "symbol_name": "serve_report.build_file_path", + "called": True, "defined": True, "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", - "symbol_name": "serve_report.build_file_path", + "fingerprint": "762e4f7d03b1bf4359c3ca364e5581402" + "39913bfabcc5aa77156460c2eb0a355", "reachable_from": [], }, ], @@ -239,14 +252,6 @@ def test_classify_reachability(self): ReachabilityStatus.NOT_REACHABLE, ) - def test_get_changed_lines(self): - data = Path(__file__).parent.parent / "data" / "reachability" - diff_text = (data / "diff-app.patch").read_text(encoding="utf-8") - - removed, added = get_changed_lines(diff_text, "app.py") - self.assertEqual(removed, [17, 18, 19, 24]) - self.assertEqual(added, [17, 18, 19, 20, 21, 22, 27, 28, 29, 30]) - def test_build_symbol_metadata_processing(self): source_code = """ class Controller: @@ -268,21 +273,24 @@ def process_data(payload): { "Controller.process_data": { "qualified_name": "Controller.process_data", - "text": "def process_data(payload):\n" - " def inner_helper():\n" - " return True\n" - " return payload.strip()", - "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" - "a8bc9054a405131adf7dc21e06e2e7c0c1", + "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce302a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", }, + "Controller.process_data.inner_helper": { + "qualified_name": "Controller.process_data.inner_helper", + "text": "def inner_helper():\n return True", + "fingerprint": "ee2e246e01e960826cb39a9466e58095d209fdd1cbf8458630be430b3371d6a3", + "start_line": 4, + "end_line": 5, + "node_type": "function_definition", + }, "process_data": { "qualified_name": "process_data", "text": "def process_data(payload):\n return payload", - "fingerprint": "9b2797712c9ab60ea8452a441396" - "5c94d1b2f63739cab7de695e7b1dc0cf439a", + "fingerprint": "9b2797712c9ab60ea8452a4413965c94d1b2f63739cab7de695e7b1dc0cf439a", "start_line": 9, "end_line": 10, "node_type": "function_definition", @@ -359,11 +367,15 @@ def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") + diff_line_map = parse_diff_lines(diff_text=diff_text) + file_path = "app.py" + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) vuln_meta, fixed_meta, lang = analyze_patched_file( vulnerable_text=vuln_text, fixed_text=fixed_text, - diff_text=diff_text, + removed_lines=removed_lines, + added_lines=added_lines, file_path="app.py", ) @@ -379,8 +391,7 @@ def test_analyze_patched_file(self): " # Helper function nested inside serve_report\n" " def build_file_path(filename):\n" " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd"' - " could read system files.\n" + ' # An attacker passing "../../etc/passwd" could read system files.\n' " return os.path.join(generator.base_dir, filename)\n\n" " if not requested_file:\n" ' return "Error: No file specified"\n\n' @@ -388,12 +399,23 @@ def test_analyze_patched_file(self): " if os.path.exists(target_path):\n" ' return f"Serving content of {target_path}"\n\n' ' return "Error: File not found"', - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", + "fingerprint": "d7675efb263896da2a3c006795118" + "33553907e7e6ea619115a6dfc8625c3457e", "start_line": 11, "end_line": 30, "node_type": "function_definition", - } + }, + "serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": "def build_file_path(filename):\n" + " # VULNERABLE: Direct concatenation allows Path Traversal\n" + ' # An attacker passing "../../etc/passwd" could read system files.\n' + " return os.path.join(generator.base_dir, filename)", + "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", + "start_line": 17, + "end_line": 20, + "node_type": "function_definition", + }, }, ) @@ -404,32 +426,20 @@ def test_analyze_patched_file(self): "qualified_name": "serve_report", "text": "def serve_report(request_payload):\n" ' """Top-level function handling a request."""\n' - ' generator = ReportGenerator("/var/reports")\n' - ' requested_file = request_payload.get("file")\n\n' - " # Helper function nested inside serve_report\n" - " def build_file_path(filename):\n" - " # FIXED: Validate that the resolved" - " path stays within the base_dir\n" - " base = os.path.abspath(generator.base_dir)\n" - " target = os.path.abspath(os.path.join(base, filename))\n" - " if not target.startswith(base):\n" - ' raise ValueError("Path Traversal Detected")\n' - " return target\n\n" - " if not requested_file:\n " - ' return "Error: No file specified"\n\n' - " try:\n" - " target_path = build_file_path(requested_file)\n" - " except ValueError:\n" - ' return "Error: Invalid path"\n\n ' - " if os.path.exists(target_path):\n" - ' return f"Serving content of {target_path}"\n\n' - ' return "Error: File not found"', - "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d7" - "3d9afdfc3f469ccf86e8835915d240e76", + ' generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d73d9afdfc3f469ccf86e8835915d240e76", "start_line": 11, "end_line": 36, "node_type": "function_definition", - } + }, + "serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": 'def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target', + "fingerprint": "646743b5d5497f6ea3b96f860bcbeb38096ce008ad16d2b9a9c3f77a98faca80", + "start_line": 17, + "end_line": 23, + "node_type": "function_definition", + }, }, ) From bf94dc81db3ef4cea4a5ac063505a0b1a7b3239c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 25 Jun 2026 15:28:07 +0300 Subject: [PATCH 10/50] Fix formatting and linting errors. Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 254 +++++++++--------- scanpipe/pipes/symbols.py | 6 +- .../tests/pipes/test_symbols_reachability.py | 94 +++++-- 3 files changed, 195 insertions(+), 159 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 4a156c9b60..880d48946b 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -250,7 +250,8 @@ def analyze_patched_file( def parse_diff_lines(diff_text) -> dict[str, tuple[list[int], list[int]]]: """ - Parses the entire unified diff text once and maps each file path to its tuple of (removed_lines, added_lines). + Parse the entire unified diff text once and map each file path + to its tuple of (removed_lines, added_lines). """ diff_map = {} if not diff_text: @@ -358,7 +359,75 @@ def collect_patch_symbols(repo, commit_hash): return by_language -def collect_and_store_symbol_reachability_results(project, logger=None): +def generate_reachability_report(patch, candidate_resources, cloned_repos, logger=None): + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + + try: + if vcs_url not in cloned_repos: + cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) + + repo_path = cloned_repos[vcs_url] + patch_symbols_by_language = collect_patch_symbols(Repo(repo_path), commit_hash) + + if not patch_symbols_by_language: + return + + for resource in candidate_resources: + resource_language = resource.programming_language + patch_symbols = patch_symbols_by_language.get(resource_language) + + if not all([patch_symbols, resource.file_content]): + continue + + resource_index = build_resource_index( + resource.file_content, + resource_language, + ) + + if not resource_index: + continue + + vuln_evidence = match_symbols_against_resource( + patch_symbols["vulnerable"], + resource_index, + ) + + fixed_evidence = match_symbols_against_resource( + patch_symbols["fixed"], + resource_index, + ) + + if not any([vuln_evidence, fixed_evidence]): + continue + + result = { + "symbols_reachability": { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability(vuln_evidence).value, + } + } + + resource.update_extra_data( + { + "symbols_reachability": result, + } + ) + except Exception as e: + if logger: + logger( + f"Failed to collect symbol reachability for " + f"{vcs_url}@{commit_hash}: {e}" + ) + + +def get_symbol_reachability_results(project, logger=None): """ For each known patch commit, determine whether each project codebase resource is reachable to the vulnerable code by comparing tree-sitter ASTs @@ -370,88 +439,13 @@ def collect_and_store_symbol_reachability_results(project, logger=None): is_media=False, ) - vulnerabilities = project.package_vulnerabilities - - for vulnerability in vulnerabilities: - patches = vulnerability.get("fixed_in_patches", []) - + for vulnerability in project.package_vulnerabilities: cloned_repos = {} - for patch in patches: - vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - - if not vcs_url or not commit_hash: - continue - - try: - if vcs_url not in cloned_repos: - cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) - - repo_path = cloned_repos[vcs_url] - repo = Repo(repo_path) - - patch_symbols_by_language = collect_patch_symbols(repo, commit_hash) - - if not patch_symbols_by_language: - continue - - for resource in candidate_resources: - resource_language = resource.programming_language - if resource_language not in patch_symbols_by_language: - continue - - resource_text = resource.file_content - if not resource_text: - continue - - patch_symbols = patch_symbols_by_language[resource_language] - resource_index = build_resource_index( - resource_text, - resource_language, - ) - - if not resource_index: - continue - - vuln_evidence = match_symbols_against_resource( - patch_symbols["vulnerable"], - resource_index, - ) - - fixed_evidence = match_symbols_against_resource( - patch_symbols["fixed"], - resource_index, - ) - - if not vuln_evidence and not fixed_evidence: - continue - - result = { - "symbols_reachability": { - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, - "evidence": list(vuln_evidence.values()), - "fixed_symbols": sorted(fixed_evidence.keys()), - "vulnerable_symbols": sorted(vuln_evidence.keys()), - "reachability_status": classify_reachability( - vuln_evidence - ).value, - } - } - - resource.update_extra_data( - { - "symbols_reachability": result, - } - ) - except Exception as e: - if logger: - logger( - f"Failed to collect symbol reachability for " - f"{vcs_url}@{commit_hash}: {e}" - ) + for patch in vulnerability.get("fixed_in_patches", []): + if patch.get("vcs_url") and patch.get("commit_hash"): + generate_reachability_report( + patch, candidate_resources, cloned_repos, logger + ) for url, path in cloned_repos.items(): try: @@ -508,7 +502,6 @@ def __init__(self, tree, language: str): self.class_methods: set[str] = set() def build(self) -> CallGraph: - """Main execution flow for building the call graph.""" self._extract_nodes() edges = self._resolve_all_edges() @@ -609,8 +602,8 @@ def match_symbols_against_resource( } reachable_callers, _ = compute_reachable_symbols( - edges_qualified, - target_qualified_names, + edges_qualified=edges_qualified, + target_qualified_names=target_qualified_names, ) called_qualified_names = set() @@ -730,11 +723,7 @@ def build_call_graph(tree, language) -> CallGraph | None: def extract_direct_calls(node, language): """ - Return direct calls inside `node`, excluding calls inside nested definitions. - - Returns: - list of (receiver_name, callee_name) - + Return direct calls inside `node` Examples: foo() -> (None, "foo") self.foo() -> ("self", "foo") @@ -779,17 +768,13 @@ def get_call_receiver(callee_node): return object_node.text.decode("utf-8", errors="replace") -def compute_reachable_symbols(call_graph, target_qualified_names): +def compute_reachable_symbols(edges_qualified, target_qualified_names): """Transitive callers using resolved qualified-name edges.""" - if not call_graph or not target_qualified_names: - return set(), False - - edges = call_graph.get("edges_qualified") - if not edges: + if not edges_qualified or not target_qualified_names: return set(), False callers_of = {} - for caller, callees in edges.items(): + for caller, callees in edges_qualified.items(): for callee in callees: callers_of.setdefault(callee, set()).add(caller) @@ -878,9 +863,49 @@ def resolve_callee( return set() +def extract_imports(captures: dict) -> tuple[str | None, list[tuple[str, str | None]]]: + """ + Flatten and sort tree-sitter captures, then extract the module name + and a list of import/alias pairs. + """ + flat_captures = [] + for tag, nodes in captures.items(): + for n in nodes: + flat_captures.append( + ( + n.start_byte, + tag, + n.text.decode("utf-8", errors="replace").strip("'\""), + ) + ) + + flat_captures.sort(key=lambda x: x[0]) + + module_name = None + current_import = None + pairs = [] + + for _, tag, text in flat_captures: + if tag == "module_name": + module_name = text + elif tag == "import_name": + if current_import is not None: + pairs.append((current_import, None)) + current_import = text + elif tag == "alias": + pairs.append((current_import, text)) + current_import = None + + if current_import is not None: + pairs.append((current_import, None)) + + return module_name, pairs + + def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: """ - Extract import statements from a Tree-sitter AST and map every local alias to its absolute imported path. + Extract import statements from a Tree-sitter AST and map every + local alias to its absolute imported path. """ config = get_imports_language_config(language) separator = config["separator"] @@ -894,36 +919,7 @@ def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: wildcard_modules: list[str] = [] for _pattern_index, captures in query.matches(root_node): - flat_captures = [] - for tag, nodes in captures.items(): - for n in nodes: - flat_captures.append( - ( - n.start_byte, - tag, - n.text.decode("utf-8", errors="replace").strip("'\""), - ) - ) - - flat_captures.sort(key=lambda x: x[0]) - - module_name = None - current_import = None - pairs = [] - - for _, tag, text in flat_captures: - if tag == "module_name": - module_name = text - elif tag == "import_name": - if current_import is not None: - pairs.append((current_import, None)) - current_import = text - elif tag == "alias": - pairs.append((current_import, text)) - current_import = None - - if current_import is not None: - pairs.append((current_import, None)) + module_name, pairs = extract_imports(captures) for imp_name, alias in pairs: if not imp_name: diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index b4dd67fb44..1cc142057a 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -209,7 +209,8 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "imports": """ (import_statement name: [ (dotted_name) @import_name - (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + (aliased_import name: (dotted_name) + @import_name alias: (identifier) @alias) ]) ; Combine explicit, relative, aliased, and wildcard from_imports @@ -220,7 +221,8 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): ] [ name: (dotted_name) @import_name - name: (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) + name: (aliased_import name: (dotted_name) + @import_name alias: (identifier) @alias) (wildcard_import) @import_name ] ) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 93937dc857..572dd782e6 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -32,11 +32,11 @@ from scanpipe.pipes.reachability import analyze_patched_file from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results from scanpipe.pipes.reachability import collect_imports from scanpipe.pipes.reachability import compute_reachable_symbols from scanpipe.pipes.reachability import diff_changed_symbols from scanpipe.pipes.reachability import extract_direct_calls +from scanpipe.pipes.reachability import get_symbol_reachability_results from scanpipe.pipes.reachability import parse_diff_lines from scanpipe.pipes.symbols import collect_definitions from scanpipe.pipes.symbols import extract_definitions @@ -55,7 +55,7 @@ def setUp(self): @patch("scanpipe.pipes.reachability.clone_repo") @patch("scanpipe.pipes.reachability.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) - def test_collect_and_store_symbol_reachability_results( + def test_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, @@ -111,7 +111,7 @@ def test_collect_and_store_symbol_reachability_results( resource.save() with patch("scanpipe.pipes.reachability.Repo"): - collect_and_store_symbol_reachability_results(self.project1) + get_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") @@ -126,21 +126,22 @@ def test_collect_and_store_symbol_reachability_results( }, "evidence": [ { - "symbol_name": "serve_report", "called": False, "defined": True, "imported": False, - "fingerprint": "d7675efb263896da2a3c00679511833553907e7e6ea619115a6dfc8625c3457e", + "fingerprint": "d7675efb263896da2a3c00679511833" + "553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", "reachable_from": [], }, { - "symbol_name": "serve_report.build_file_path", "called": True, "defined": True, "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364e5581402" - "39913bfabcc5aa77156460c2eb0a355", - "reachable_from": [], + "fingerprint": "762e4f7d03b1bf4359c3ca364" + "e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": ["serve_report"], }, ], "fixed_symbols": ["serve_report", "serve_report.build_file_path"], @@ -273,8 +274,12 @@ def process_data(payload): { "Controller.process_data": { "qualified_name": "Controller.process_data", - "text": "def process_data(payload):\n def inner_helper():\n return True\n return payload.strip()", - "fingerprint": "b0d0ad9a92209a6d79b84e932ce302a8bc9054a405131adf7dc21e06e2e7c0c1", + "text": "def process_data(payload):\n" + " def inner_helper():\n" + " return True\n " + " return payload.strip()", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" + "a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", @@ -282,7 +287,8 @@ def process_data(payload): "Controller.process_data.inner_helper": { "qualified_name": "Controller.process_data.inner_helper", "text": "def inner_helper():\n return True", - "fingerprint": "ee2e246e01e960826cb39a9466e58095d209fdd1cbf8458630be430b3371d6a3", + "fingerprint": "ee2e246e01e960826cb39a9466e5" + "8095d209fdd1cbf8458630be430b3371d6a3", "start_line": 4, "end_line": 5, "node_type": "function_definition", @@ -290,7 +296,8 @@ def process_data(payload): "process_data": { "qualified_name": "process_data", "text": "def process_data(payload):\n return payload", - "fingerprint": "9b2797712c9ab60ea8452a4413965c94d1b2f63739cab7de695e7b1dc0cf439a", + "fingerprint": "9b2797712c9ab60ea8452a4413965" + "c94d1b2f63739cab7de695e7b1dc0cf439a", "start_line": 9, "end_line": 10, "node_type": "function_definition", @@ -391,7 +398,9 @@ def test_analyze_patched_file(self): " # Helper function nested inside serve_report\n" " def build_file_path(filename):\n" " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd" could read system files.\n' + " # An attacker passing " + '"../../etc/passwd" ' + "could read system files.\n" " return os.path.join(generator.base_dir, filename)\n\n" " if not requested_file:\n" ' return "Error: No file specified"\n\n' @@ -409,9 +418,11 @@ def test_analyze_patched_file(self): "qualified_name": "serve_report.build_file_path", "text": "def build_file_path(filename):\n" " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd" could read system files.\n' + ' # An attacker passing "../../etc/passwd"' + " could read system files.\n" " return os.path.join(generator.base_dir, filename)", - "fingerprint": "762e4f7d03b1bf4359c3ca364e558140239913bfabcc5aa77156460c2eb0a355", + "fingerprint": "762e4f7d03b1bf4359c3ca364e5581" + "40239913bfabcc5aa77156460c2eb0a355", "start_line": 17, "end_line": 20, "node_type": "function_definition", @@ -426,16 +437,44 @@ def test_analyze_patched_file(self): "qualified_name": "serve_report", "text": "def serve_report(request_payload):\n" ' """Top-level function handling a request."""\n' - ' generator = ReportGenerator("/var/reports")\n requested_file = request_payload.get("file")\n\n # Helper function nested inside serve_report\n def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target\n\n if not requested_file:\n return "Error: No file specified"\n\n try:\n target_path = build_file_path(requested_file)\n except ValueError:\n return "Error: Invalid path"\n\n if os.path.exists(target_path):\n return f"Serving content of {target_path}"\n\n return "Error: File not found"', - "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d73d9afdfc3f469ccf86e8835915d240e76", + ' generator = ReportGenerator("/var/reports")\n' + ' requested_file = request_payload.get("file")\n\n' + " # Helper function nested inside serve_report\n" + " def build_file_path(filename):\n" + " # FIXED: Validate that the resolved " + "path stays within the base_dir\n" + " base = os.path.abspath(generator.base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n" + ' raise ValueError("Path Traversal Detected")\n' + " return target\n\n" + " if not requested_file:\n" + ' return "Error: No file specified"\n\n' + " try:\n" + " target_path = build_file_path(requested_file)\n" + " except ValueError:\n" + ' return "Error: Invalid path"\n\n' + " if os.path.exists(target_path):\n" + ' return f"Serving content of {target_path}"\n\n' + ' return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d" + "73d9afdfc3f469ccf86e8835915d240e76", "start_line": 11, "end_line": 36, "node_type": "function_definition", }, "serve_report.build_file_path": { "qualified_name": "serve_report.build_file_path", - "text": 'def build_file_path(filename):\n # FIXED: Validate that the resolved path stays within the base_dir\n base = os.path.abspath(generator.base_dir)\n target = os.path.abspath(os.path.join(base, filename))\n if not target.startswith(base):\n raise ValueError("Path Traversal Detected")\n return target', - "fingerprint": "646743b5d5497f6ea3b96f860bcbeb38096ce008ad16d2b9a9c3f77a98faca80", + "text": "def build_file_path(filename):\n" + " # FIXED: Validate that the resolved path" + " stays within the base_dir\n" + " base = os.path.abspath(generator.base_dir)\n" + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n" + ' raise ValueError("Path Traversal Detected")\n' + " return target", + "fingerprint": "646743b5d5497f6ea3b96f860bcbe" + "b38096ce008ad16d2b9a9c3f77a98faca80", "start_line": 17, "end_line": 23, "node_type": "function_definition", @@ -480,17 +519,16 @@ def test_extract_symbols_deduplication(self): self.assertEqual(enclosing_symbols[0].type, "function_definition") def test_compute_reachable_symbols(self): - call_graph = { - "edges_qualified": { - "app.main": {"app.helper", "app.safe_func"}, - "app.helper": {"app.vuln_func"}, - "app.direct_caller": {"app.vuln_func"}, - "app.unrelated": {"app.safe_func"}, - } + edges_qualified = { + "app.main": {"app.helper", "app.safe_func"}, + "app.helper": {"app.vuln_func"}, + "app.direct_caller": {"app.vuln_func"}, + "app.unrelated": {"app.safe_func"}, } target_qns = ["app.vuln_func"] - reachable, has_direct = compute_reachable_symbols(call_graph, target_qns) + reachable, has_direct = compute_reachable_symbols(edges_qualified, target_qns) + self.assertTrue(has_direct) expected_reachable = {"app.main", "app.helper", "app.direct_caller"} From c9b96527d31369d773670b347bf51556c48d99c9 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Sat, 27 Jun 2026 00:29:07 +0300 Subject: [PATCH 11/50] Refactor and simplify reachability pipeline Improve pipeline structure Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 1133 ++++++----------- scanpipe/pipes/symbols.py | 355 ++++-- .../tests/pipes/test_symbols_reachability.py | 288 +++-- 3 files changed, 791 insertions(+), 985 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 880d48946b..b6bf86c837 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -23,29 +23,19 @@ import os import shutil import tempfile -from dataclasses import dataclass -from dataclasses import field from enum import Enum from pathlib import Path from typing import Any from git import Repo from git.diff import NULL_TREE -from git.exc import BadName from scancode.api import get_file_info from unidiff import PatchSet from scanpipe.pipes.symbols import TS_QUERIES -from scanpipe.pipes.symbols import _root_of -from scanpipe.pipes.symbols import collect_definitions +from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint -from scanpipe.pipes.symbols import extract_definitions -from scanpipe.pipes.symbols import extract_symbols -from scanpipe.pipes.symbols import get_query -from scanpipe.pipes.symbols import parse_code_to_ast -from scanpipe.pipes.symbols import qualified_name_from_index - -EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" +from scanpipe.pipes.symbols import is_supported_language class ReachabilityStatus(str, Enum): @@ -54,31 +44,6 @@ class ReachabilityStatus(str, Enum): NOT_REACHABLE = "NOT_REACHABLE" -def clone_repo(vcs_url, commit_hash=None): - repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") - - try: - repo = Repo.clone_from(vcs_url, repo_path) - - if commit_hash: - repo.git.checkout(commit_hash) - - return repo_path - - except BadName as exc: - cleanup_repo(repo_path) - raise ValueError(f"Commit {commit_hash} not found") from exc - - except Exception: - cleanup_repo(repo_path) - raise - - -def cleanup_repo(repo_path): - if repo_path and os.path.exists(repo_path): - shutil.rmtree(repo_path, ignore_errors=True) - - def normalize_text(content): if content is None: return "" @@ -89,17 +54,7 @@ def normalize_text(content): return str(content) -def is_supported_language(language): - """Return True if the language is supported by tree-sitter queries.""" - return bool(language) and language in TS_QUERIES - - def detect_language_with_scancode(file_path, content): - """ - Write `content` to a temp file preserving `file_path`'s basename - so the extension is meaningful, then ask ScanCode's `get_file_info` - to return the programming language. - """ content = normalize_text(content) if not content: @@ -118,257 +73,287 @@ def detect_language_with_scancode(file_path, content): shutil.rmtree(tmp_dir, ignore_errors=True) -def get_commit_and_parent(repo, commit_hash): - commit = repo.commit(commit_hash) - parent = commit.parents[0] if commit.parents else None - return commit, parent +class GitRepositoryContext: + def __init__(self, vcs_url: str, commit_hash: str = None): + self.vcs_url = vcs_url + self.commit_hash = commit_hash + self.repo_path = None + self._repo = None + + def __enter__(self) -> "GitRepositoryContext": + self.repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") + try: + self._repo = Repo.clone_from(self.vcs_url, self.repo_path) + if self.commit_hash: + self._repo.git.checkout(self.commit_hash) + return self + except Exception as exc: + self._cleanup() + raise ValueError( + f"Failed to clone/checkout {self.vcs_url}@{self.commit_hash}" + ) from exc + + def __exit__(self, exc_type, exc_val, exc_tb): + self._cleanup() + + @property + def repo(self) -> Repo: + return self._repo + + def _cleanup(self): + if self.repo_path and os.path.exists(self.repo_path): + shutil.rmtree(self.repo_path, ignore_errors=True) + + +class PatchAnalyzer: + EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" + + def __init__(self, repo: Repo, commit_hash: str): + self.repo = repo + self.commit = repo.commit(commit_hash) + self.parent_commit = self.commit.parents[0] if self.commit.parents else None + + def get_changed_files(self): + diffs = ( + self.parent_commit.diff(self.commit, create_patch=False) + if self.parent_commit + else self.commit.diff(NULL_TREE, create_patch=False) + ) + files = {} + for diff in diffs: + change_type = diff.change_type + old_path = diff.a_path if change_type in ("D", "M", "R") else None + new_path = diff.b_path if change_type in ("A", "M", "R") else None + path_key = new_path or old_path -def get_commit_diff_text(repo, parent_commit, commit): - """Whole-commit unified diff (used to extract changed line numbers).""" - base = parent_commit.hexsha if parent_commit else EMPTY_TREE_SHA - return repo.git.diff(base, commit.hexsha, unified=3) + if not path_key: + continue + entry = files.setdefault( + path_key, {"vulnerable_text": "", "fixed_text": ""} + ) -def get_changed_files(parent_commit, commit): - """ - Return: - { - file_path: { - "vulnerable_text": "...", - "fixed_text": "...", - } - } + if old_path and self.parent_commit: + entry["vulnerable_text"] = ( + (self.parent_commit.tree / old_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) - """ - diffs = ( - parent_commit.diff(commit, create_patch=False) - if parent_commit - else commit.diff(NULL_TREE, create_patch=False) - ) + if new_path: + entry["fixed_text"] = ( + (self.commit.tree / new_path) + .data_stream.read() + .decode("utf-8", errors="replace") + ) - files = {} - for diff in diffs: - change_type = diff.change_type - old_path = diff.a_path if change_type in ("D", "M", "R") else None - new_path = diff.b_path if change_type in ("A", "M", "R") else None - path_key = new_path or old_path - - if not path_key: - continue - - entry = files.setdefault( - path_key, - { - "vulnerable_text": "", - "fixed_text": "", - }, - ) + return files - if old_path and parent_commit: - entry["vulnerable_text"] = ( - (parent_commit.tree / old_path) - .data_stream.read() - .decode("utf-8", errors="replace") - ) + def get_commit_diff_text(self): + base = self.parent_commit.hexsha if self.parent_commit else self.EMPTY_TREE_SHA + return self.repo.git.diff(base, self.commit.hexsha, unified=3) - if new_path: - entry["fixed_text"] = ( - (commit.tree / new_path) - .data_stream.read() - .decode("utf-8", errors="replace") - ) + @classmethod + def diff_changed_symbols(cls, vuln_meta, fixed_meta): + vuln_only = { + key: metadata + for key, metadata in vuln_meta.items() + if fixed_meta.get(key, {}).get("text") != metadata["text"] + } + fixed_only = { + key: metadata + for key, metadata in fixed_meta.items() + if vuln_meta.get(key, {}).get("text") != metadata["text"] + } + return vuln_only, fixed_only + + @classmethod + def parse_diff_lines(cls, diff_text): + diff_map = {} + if not diff_text: + return diff_map + + for patched_file in PatchSet.from_string(diff_text): + removed = [] + added = [] + + for hunk in patched_file: + hunk_removed = [ + line.source_line_no + for line in hunk + if line.is_removed and line.source_line_no + ] + hunk_added = [ + line.target_line_no + for line in hunk + if line.is_added and line.target_line_no + ] + + if hunk_added and not hunk_removed: + anchor = max(hunk.source_start, 1) + hunk_removed = [anchor] + + if hunk_removed and not hunk_added: + anchor = max(hunk.target_start, 1) + hunk_added = [anchor] + + removed.extend(hunk_removed) + added.extend(hunk_added) + + candidates = { + patched_file.path, + (patched_file.source_file or "").removeprefix("a/"), + (patched_file.target_file or "").removeprefix("b/"), + } - return files - - -def diff_changed_symbols(vuln_meta, fixed_meta): - """ - Keep only symbols whose body actually differs between vulnerable and fixed - versions. Utilizes the unique suffix keys generated by build_symbol_metadata. - """ - vuln_only = { - key: metadata - for key, metadata in vuln_meta.items() - if fixed_meta.get(key, {}).get("text") != metadata["text"] - } - - fixed_only = { - key: metadata - for key, metadata in fixed_meta.items() - if vuln_meta.get(key, {}).get("text") != metadata["text"] - } - - return vuln_only, fixed_only - - -def analyze_patched_file( - vulnerable_text, fixed_text, removed_lines, added_lines, file_path -): - """ - Return `(vuln_metadata, fixed_metadata, language)` for one changed file, - restricted to symbols actually touched by the patch. - """ - vulnerable_text = normalize_text(vulnerable_text) - fixed_text = normalize_text(fixed_text) - - language = detect_language_with_scancode( - file_path, fixed_text - ) or detect_language_with_scancode(file_path, vulnerable_text) - - if not is_supported_language(language): - return {}, {}, language - - vuln_tree, _ = ( - parse_code_to_ast(vulnerable_text, language) - if vulnerable_text - else (None, None) - ) + for candidate in candidates: + if candidate: + diff_map[candidate] = (removed, added) - fixed_tree, _ = ( - parse_code_to_ast(fixed_text, language) if fixed_text else (None, None) - ) + return diff_map - if vuln_tree is None and fixed_tree is None: - return {}, {}, language + def collect_patch_symbols(self): + diff_text = self.get_commit_diff_text() + changed_files = self.get_changed_files() + diff_line_map = self.parse_diff_lines(diff_text=diff_text) + + by_language = {} + for file_path, texts in changed_files.items(): + vulnerable_text = texts["vulnerable_text"] + fixed_text = texts["fixed_text"] + removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + + vuln_meta, fixed_meta, language = self.analyze( + vulnerable_text=vulnerable_text, + fixed_text=fixed_text, + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, + ) - vuln_nodes = ( - extract_symbols(vuln_tree, removed_lines, language) if vuln_tree else [] - ) + if not language or not (vuln_meta or fixed_meta): + continue - fixed_nodes = ( - extract_symbols(fixed_tree, added_lines, language) if fixed_tree else [] - ) + language_bucket = by_language.setdefault( + language, {"vulnerable": {}, "fixed": {}} + ) - vuln_meta, fixed_meta = diff_changed_symbols( - build_symbol_metadata(vuln_nodes, language), - build_symbol_metadata(fixed_nodes, language), - ) + language_bucket["vulnerable"].update( + {f"{file_path}::{key}": metadata for key, metadata in vuln_meta.items()} + ) + language_bucket["fixed"].update( + { + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() + } + ) - return vuln_meta, fixed_meta, language + return by_language + @classmethod + def build_symbol_metadata( + cls, nodes, extractor: SymbolExtractor, index: dict = None + ): + if not nodes or not extractor: + return {} -def parse_diff_lines(diff_text) -> dict[str, tuple[list[int], list[int]]]: - """ - Parse the entire unified diff text once and map each file path - to its tuple of (removed_lines, added_lines). - """ - diff_map = {} - if not diff_text: - return diff_map + if index is None: + index = extractor.extract_definitions_index() - for patched_file in PatchSet.from_string(diff_text): - removed = [] - added = [] - - for hunk in patched_file: - hunk_removed = [ - line.source_line_no - for line in hunk - if line.is_removed and line.source_line_no - ] - hunk_added = [ - line.target_line_no - for line in hunk - if line.is_added and line.target_line_no - ] - - # Pure insertion anchor - if hunk_added and not hunk_removed: - anchor = max(hunk.source_start, 1) - hunk_removed = [anchor] - - # Pure deletion anchor - if hunk_removed and not hunk_added: - anchor = max(hunk.target_start, 1) - hunk_added = [anchor] - - removed.extend(hunk_removed) - added.extend(hunk_added) - - # Register the line tracking data against all naming variations of this file - candidates = { - patched_file.path, - (patched_file.source_file or "").removeprefix("a/"), - (patched_file.target_file or "").removeprefix("b/"), - } + metadata = {} + for node in nodes: + qualified_name = extractor._build_qualified_name(node, index) + if not qualified_name: + continue - for candidate in candidates: - if candidate: - diff_map[candidate] = (removed, added) + body_text = node.text.decode("utf-8", errors="replace") + fingerprints = create_sha256_fingerprint(body_text) + + key = qualified_name + suffix = 1 + while key in metadata: + suffix += 1 + key = f"{qualified_name}#{suffix}" + + metadata[key] = { + "qualified_name": qualified_name, + "text": body_text, + "fingerprint": fingerprints, + "start_line": node.start_point[0] + 1, + "end_line": node.end_point[0] + 1, + "node_type": node.type, + } + return metadata - return diff_map + @classmethod + def analyze( + self, vulnerable_text, fixed_text, removed_lines, added_lines, file_path + ): + vulnerable_text = normalize_text(vulnerable_text) + fixed_text = normalize_text(fixed_text) + language = detect_language_with_scancode( + file_path, fixed_text + ) or detect_language_with_scancode(file_path, vulnerable_text) -def collect_patch_symbols(repo, commit_hash): - """ - Return: - { - language: { - "vulnerable": { - "file_path::symbol_key": metadata, - ... - }, - "fixed": { - "file_path::symbol_key": metadata, - ... - }, - }, - ... - } + if not is_supported_language(language): + return {}, {}, language - """ - commit, parent = get_commit_and_parent(repo=repo, commit_hash=commit_hash) - diff_text = get_commit_diff_text(repo=repo, parent_commit=parent, commit=commit) - changed = get_changed_files(parent_commit=parent, commit=commit) - diff_line_map = parse_diff_lines(diff_text=diff_text) - - by_language = {} - for file_path, texts in changed.items(): - vulnerable_text = texts["vulnerable_text"] - fixed_text = texts["fixed_text"] - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) - - vuln_meta, fixed_meta, language = analyze_patched_file( - vulnerable_text=vulnerable_text, - fixed_text=fixed_text, - removed_lines=removed_lines, - added_lines=added_lines, - file_path=file_path, + lang_query = TS_QUERIES[language]() + + vuln_tree, _ = ( + lang_query.parse_code_to_ast(code_text=vulnerable_text) + if vulnerable_text + else (None, None) + ) + fixed_tree, _ = ( + lang_query.parse_code_to_ast(code_text=fixed_text) + if fixed_text + else (None, None) ) - if not language or not (vuln_meta or fixed_meta): - continue + if vuln_tree is None and fixed_tree is None: + return {}, {}, language - language_bucket = by_language.setdefault( - language, - { - "vulnerable": {}, - "fixed": {}, - }, - ) + vuln_meta_all = {} + fixed_meta_all = {} - language_bucket["vulnerable"].update( - {f"{file_path}::{key}": metadata for key, metadata in vuln_meta.items()} - ) + if vuln_tree: + vuln_extractor = SymbolExtractor( + lang_query=lang_query, root_node=vuln_tree.root_node + ) + vuln_nodes = vuln_extractor.extract_changed_symbols( + changed_lines=removed_lines + ) + vuln_meta_all = self.build_symbol_metadata( + nodes=vuln_nodes, extractor=vuln_extractor + ) - language_bucket["fixed"].update( - {f"{file_path}::{key}": metadata for key, metadata in fixed_meta.items()} - ) + if fixed_tree: + fixed_extractor = SymbolExtractor( + lang_query=lang_query, root_node=fixed_tree.root_node + ) + fixed_nodes = fixed_extractor.extract_changed_symbols( + changed_lines=added_lines + ) + fixed_meta_all = self.build_symbol_metadata( + fixed_nodes, extractor=fixed_extractor + ) - return by_language + vuln_meta, fixed_meta = self.diff_changed_symbols( + vuln_meta=vuln_meta_all, fixed_meta=fixed_meta_all + ) + return vuln_meta, fixed_meta, language -def generate_reachability_report(patch, candidate_resources, cloned_repos, logger=None): +def generate_reachability_report(patch, repo, candidate_resources, logger=None): vcs_url = patch.get("vcs_url") commit_hash = patch.get("commit_hash") try: - if vcs_url not in cloned_repos: - cloned_repos[vcs_url] = clone_repo(vcs_url, commit_hash) - - repo_path = cloned_repos[vcs_url] - patch_symbols_by_language = collect_patch_symbols(Repo(repo_path), commit_hash) + patch_analyzer = PatchAnalyzer(repo=repo, commit_hash=commit_hash) + patch_symbols_by_language = patch_analyzer.collect_patch_symbols() if not patch_symbols_by_language: return @@ -377,31 +362,29 @@ def generate_reachability_report(patch, candidate_resources, cloned_repos, logge resource_language = resource.programming_language patch_symbols = patch_symbols_by_language.get(resource_language) - if not all([patch_symbols, resource.file_content]): + if not patch_symbols: + continue + + file_content = normalize_text(resource.file_content) + if not file_content: continue - resource_index = build_resource_index( - resource.file_content, - resource_language, + resource_analyzer = ResourceAnalyzer( + resource_text=file_content, language=resource_language ) + resource_index = resource_analyzer.build_index() if not resource_index: continue - vuln_evidence = match_symbols_against_resource( - patch_symbols["vulnerable"], - resource_index, - ) - - fixed_evidence = match_symbols_against_resource( - patch_symbols["fixed"], - resource_index, - ) + matcher = ResourcePatchMatcher(resource_index=resource_index) + vuln_evidence = matcher.match(patch_symbols["vulnerable"]) + fixed_evidence = matcher.match(patch_symbols["fixed"]) if not any([vuln_evidence, fixed_evidence]): continue - result = { + report = { "symbols_reachability": { "patch": { "vcs_url": vcs_url, @@ -413,12 +396,17 @@ def generate_reachability_report(patch, candidate_resources, cloned_repos, logge "reachability_status": classify_reachability(vuln_evidence).value, } } + print(report) + + existing = resource.extra_data.get("symbols_reachability") + reports = existing if isinstance(existing, list) else [] + + if not any( + r.get("patch", {}).get("commit_hash") == commit_hash for r in reports + ): + reports.append(report) + resource.update_extra_data({"symbols_reachability": reports}) - resource.update_extra_data( - { - "symbols_reachability": result, - } - ) except Exception as e: if logger: logger( @@ -427,239 +415,34 @@ def generate_reachability_report(patch, candidate_resources, cloned_repos, logge ) -def get_symbol_reachability_results(project, logger=None): - """ - For each known patch commit, determine whether each project codebase - resource is reachable to the vulnerable code by comparing tree-sitter ASTs - of the patch versus the resource. - """ +def collect_and_store_symbol_reachability_results(project, logger=None): candidate_resources = project.codebaseresources.files().filter( - is_binary=False, - is_archive=False, - is_media=False, + is_binary=False, is_archive=False, is_media=False ) for vulnerability in project.package_vulnerabilities: - cloned_repos = {} + patches_by_repo = {} for patch in vulnerability.get("fixed_in_patches", []): - if patch.get("vcs_url") and patch.get("commit_hash"): - generate_reachability_report( - patch, candidate_resources, cloned_repos, logger - ) + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + if vcs_url and commit_hash: + patches_by_repo.setdefault(vcs_url, []).append(patch) - for url, path in cloned_repos.items(): + for vcs_url, patches in patches_by_repo.items(): try: - cleanup_repo(path) + with GitRepositoryContext(vcs_url) as git_context: + for patch in patches: + commit_hash = patch.get("commit_hash") + git_context.repo.git.checkout(commit_hash) + generate_reachability_report( + patch, git_context.repo, candidate_resources, logger + ) except Exception as e: if logger: - logger(f"Failed to clean up repo {url} at {path}: {e}") - - -@dataclass -class SymbolNode: - qualified_name: str - node: Any # tree-sitter node - node_type: str - fingerprint: str - - -@dataclass -class CallGraph: - nodes: dict[str, SymbolNode] = field(default_factory=dict) - edges_qualified: dict[str, set[str]] = field(default_factory=dict) - imports: dict[str, str] = field(default_factory=dict) - - -@dataclass -class ResourceIndex: - definitions: set[str] = field(default_factory=set) - fingerprints: set[str] = field(default_factory=set) - call_graph: CallGraph | None = None - - def to_dict(self): - return { - "definitions": self.definitions, - "fingerprints": self.fingerprints, - "call_graph": { - "nodes": {k: vars(v) for k, v in self.call_graph.nodes.items()}, - "edges_qualified": self.call_graph.edges_qualified, - "imports": self.call_graph.imports, - } - if self.call_graph - else None, - } - - -class CallGraphBuilder: - def __init__(self, tree, language: str): - self.tree = tree - self.language = language - self.index = collect_definitions(tree.root_node, language) - self.imports = collect_imports(tree.root_node, language) - - self.nodes: dict[str, SymbolNode] = {} - self.definitions_by_name: dict[str, set[str]] = {} - self.class_methods: set[str] = set() - - def build(self) -> CallGraph: - self._extract_nodes() - edges = self._resolve_all_edges() - - return CallGraph(nodes=self.nodes, edges_qualified=edges, imports=self.imports) - - def _extract_nodes(self): - """Extract all definitions and populate lookup maps.""" - sep = get_imports_language_config(self.language)["separator"] - - for definition in self.index.values(): - node = definition["node"] - qualified_name = qualified_name_from_index(node, self.index) - - if not qualified_name: - continue - - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_sha256_fingerprint(body_text) - - self.nodes[qualified_name] = SymbolNode( - qualified_name=qualified_name, - node=node, - node_type=node.type, - fingerprint=fingerprint, - ) - - short_name = qualified_name.rsplit(sep, 1)[-1] - self.definitions_by_name.setdefault(short_name, set()).add(qualified_name) - - if node.type == "function_definition": - parent = node.parent - if parent is not None and parent.type == "class_definition": - self.class_methods.add(qualified_name) - - def _resolve_all_edges(self) -> dict[str, set[str]]: - """Map out all direct calls for every node in the graph.""" - edges_qualified = {} - - for qualified_name, symbol_node in self.nodes.items(): - direct_calls = extract_direct_calls(symbol_node.node, self.language) - resolved_callees = set() - - for receiver_name, callee_name in direct_calls: - resolved_callees |= resolve_callee( - receiver_name=receiver_name, - callee_name=callee_name, - owner_qn=qualified_name, - definitions_by_name=self.definitions_by_name, - class_methods=self.class_methods, - import_map=self.imports, - language=self.language, - ) - - edges_qualified[qualified_name] = resolved_callees - - return edges_qualified - - -def build_resource_index(resource_text, language) -> ResourceIndex | None: - if not is_supported_language(language) or not resource_text: - return None - - tree, _ = parse_code_to_ast(resource_text, language) - if tree is None: - return None - - call_graph = build_call_graph(tree, language) - - if call_graph: - definitions = set(call_graph.nodes.keys()) - fingerprints = { - node.fingerprint for node in call_graph.nodes.values() if node.fingerprint - } - else: - meta = build_symbol_metadata(extract_definitions(tree, language), language) - definitions = {m["qualified_name"] for m in meta.values()} - fingerprints = {m["fingerprint"] for m in meta.values() if m["fingerprint"]} - - return ResourceIndex( - definitions=definitions, fingerprints=fingerprints, call_graph=call_graph - ) - - -def match_symbols_against_resource( - patch_symbols_metadata: dict[str, Any], resource_index: "ResourceIndex" -) -> dict[str, Any]: - if not patch_symbols_metadata or not resource_index: - return {} - - call_graph = resource_index.call_graph - imports = call_graph.imports if call_graph else {} - edges_qualified = call_graph.edges_qualified if call_graph else {} - - imported_fq_names = set(imports.values()) - - target_qualified_names = { - metadata["qualified_name"] for metadata in patch_symbols_metadata.values() - } - - reachable_callers, _ = compute_reachable_symbols( - edges_qualified=edges_qualified, - target_qualified_names=target_qualified_names, - ) - - called_qualified_names = set() - for callees in edges_qualified.values(): - called_qualified_names |= set(callees) - - matched = {} - for metadata in patch_symbols_metadata.values(): - qualified_name = metadata["qualified_name"] - fingerprint = metadata["fingerprint"] - - defined = qualified_name in resource_index.definitions - fingerprint_hit = bool( - fingerprint and fingerprint in resource_index.fingerprints - ) - - imported = ( - qualified_name in imports - or qualified_name in imported_fq_names - or any( - fq == qualified_name or fq.endswith("." + qualified_name) - for fq in imported_fq_names - ) - ) - - called = any( - fq_name == qualified_name or fq_name.endswith("." + qualified_name) - for fq_name in called_qualified_names - ) - - if not (defined or fingerprint_hit or called or imported): - continue - - entry = matched.setdefault( - qualified_name, - { - "symbol_name": qualified_name, - "called": False, - "defined": False, - "imported": False, - "fingerprint": None, - "reachable_from": [], - }, - ) - - if defined: - entry["defined"] = True - if imported: - entry["imported"] = True - if called: - entry["called"] = True - entry["reachable_from"] = sorted(reachable_callers) - if fingerprint: - entry["fingerprint"] = fingerprint - - return matched + logger( + f"Failed to process repository {vcs_url} " + f"for symbol reachability: {e}" + ) def classify_reachability(evidence): @@ -683,264 +466,126 @@ def classify_reachability(evidence): return status -def build_symbol_metadata(nodes, language, index=None): - if index is None and nodes: - index = collect_definitions(_root_of(nodes[0]), language) - - metadata = {} - for node in nodes: - qualified_name = qualified_name_from_index(node, index) - if not qualified_name: - continue - - body_text = node.text.decode("utf-8", errors="replace") - fingerprints = create_sha256_fingerprint(body_text) - - key = qualified_name - suffix = 1 - while key in metadata: - suffix += 1 - key = f"{qualified_name}#{suffix}" - - metadata[key] = { - "qualified_name": qualified_name, - "text": body_text, - "fingerprint": fingerprints, - "start_line": node.start_point[0] + 1, - "end_line": node.end_point[0] + 1, - "node_type": node.type, - } - return metadata - - -def build_call_graph(tree, language) -> CallGraph | None: - if tree is None or not is_supported_language(language): - return None - - builder = CallGraphBuilder(tree, language) - return builder.build() - - -def extract_direct_calls(node, language): - """ - Return direct calls inside `node` - Examples: - foo() -> (None, "foo") - self.foo() -> ("self", "foo") - obj.foo() -> ("obj", "foo") +class ResourceAnalyzer: + def __init__(self, resource_text: str, language: str): + self.resource_text = resource_text + self.language = language - """ - query = get_query(language, "calls") - if query is None or node is None: - return [] + def build_index(self) -> dict | None: + text = normalize_text(self.resource_text) + if not is_supported_language(self.language) or not text: + return None - calls = [] + lang_query = TS_QUERIES[self.language]() + tree, _ = lang_query.parse_code_to_ast(text) - for _, captures in query.matches(node): - for callee_node in captures.get("callee", []): - receiver_name = get_call_receiver(callee_node) - callee_name = callee_node.text.decode("utf-8", errors="replace") + if tree is None: + return None - if callee_name: - calls.append((receiver_name, callee_name)) - return calls + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + definitions_index = extractor.extract_definitions_index() + imports_map = extractor.extract_imports() + separator = extractor.syntax_config.get("separator", ".") + definitions = set() + fingerprints = set() + callers_of = {} # callee_name -> set of caller_qualified_names -def get_call_receiver(callee_node): - """ - Return receiver name for attribute calls. + for node, _ in lang_query.get_functions(tree.root_node): + qualified_name = extractor._build_qualified_name(node, definitions_index) + if not qualified_name: + continue - Examples: - foo() -> None - self.foo() -> "self" - obj.foo() -> "obj" + definitions.add(qualified_name) + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + if fingerprint: + fingerprints.add(fingerprint) - """ - parent = callee_node.parent + for _, callee_name in extractor.extract_calls(node): + callers_of.setdefault(callee_name, set()).add(qualified_name) - if parent is None or parent.type != "attribute": - return None + for node, _ in lang_query.get_classes(tree.root_node): + qualified_name = extractor._build_qualified_name(node, definitions_index) + if not qualified_name: + continue - object_node = parent.child_by_field_name("object") - if object_node is None: - return None + definitions.add(qualified_name) + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + if fingerprint: + fingerprints.add(fingerprint) - return object_node.text.decode("utf-8", errors="replace") - - -def compute_reachable_symbols(edges_qualified, target_qualified_names): - """Transitive callers using resolved qualified-name edges.""" - if not edges_qualified or not target_qualified_names: - return set(), False - - callers_of = {} - for caller, callees in edges_qualified.items(): - for callee in callees: - callers_of.setdefault(callee, set()).add(caller) - - targets = set(target_qualified_names) - direct = set() - for target in targets: - direct |= callers_of.get(target, set()) - - reachable = set(direct) - frontier = list(direct) - while frontier: - cur = frontier.pop() - for parent in callers_of.get(cur, ()): - if parent not in reachable: - reachable.add(parent) - frontier.append(parent) - - return reachable, bool(direct) - - -def get_imports_language_config(language: str) -> dict: - configs = { - "Python": {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"}, - "Javascript": { - "self_keyword": "this", - "separator": ".", - "wildcard_symbol": "*", - }, - "Java": {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"}, - "C++": {"self_keyword": "this", "separator": "::", "wildcard_symbol": None}, - "PHP": {"self_keyword": "$this", "separator": "::", "wildcard_symbol": None}, - "Go": {"self_keyword": None, "separator": "/", "wildcard_symbol": None}, - "Ruby": {"self_keyword": "self", "separator": "::", "wildcard_symbol": None}, - } - return configs.get( - language, {"self_keyword": None, "separator": ".", "wildcard_symbol": None} - ) + return { + "definitions": definitions, + "fingerprints": fingerprints, + "imports": imports_map, + "callers_of": callers_of, + "separator": separator, + } -def resolve_callee( - receiver_name: str | None, - callee_name: str, - owner_qn: str, - definitions_by_name: dict[str, set[str]], - class_methods: set[str], - language: str, - import_map: dict | None = None, -) -> set[str]: - config = get_imports_language_config(language) - sep = config["separator"] - self_kw = config["self_keyword"] - import_map = import_map or {} - - if receiver_name: - receiver_name = receiver_name.strip("'\"") - if callee_name: - callee_name = callee_name.strip("'\"") - - if self_kw is not None and receiver_name == self_kw: - owner_class = owner_qn.rsplit(sep, 1)[0] if sep in owner_qn else owner_qn - return {f"{owner_class}{sep}{callee_name}"} - - if callee_name in import_map: - return {import_map[callee_name]} - - if receiver_name is not None and receiver_name != self_kw: - candidates = set() - - if receiver_name in import_map and receiver_name != "*": - base = import_map[receiver_name] - candidates.add(f"{base}{sep}{callee_name}") - - static_fqn = f"{receiver_name}{sep}{callee_name}" - if static_fqn in class_methods: - candidates.add(static_fqn) - - return candidates - - local_defs = definitions_by_name.get(callee_name, set()) - if local_defs: - return local_defs - - if "*" in import_map: - return {f"{mod}{sep}{callee_name}" for mod in import_map["*"]} - - return set() - - -def extract_imports(captures: dict) -> tuple[str | None, list[tuple[str, str | None]]]: - """ - Flatten and sort tree-sitter captures, then extract the module name - and a list of import/alias pairs. - """ - flat_captures = [] - for tag, nodes in captures.items(): - for n in nodes: - flat_captures.append( - ( - n.start_byte, - tag, - n.text.decode("utf-8", errors="replace").strip("'\""), - ) +class ResourcePatchMatcher: + def __init__(self, resource_index: dict): + self.resource_index = resource_index + self.definitions = resource_index.get("definitions", set()) + self.fingerprints = resource_index.get("fingerprints", set()) + self.imports = resource_index.get("imports", {}) + self.callers_of = resource_index.get("callers_of", {}) + self.separator = resource_index.get("separator", ".") + + def match(self, patch_symbols_metadata: dict[str, Any]) -> dict[str, Any]: + if not patch_symbols_metadata or not self.resource_index: + return {} + + matched = {} + for metadata in patch_symbols_metadata.values(): + qualified_name = metadata["qualified_name"] + fingerprint = metadata["fingerprint"] + + short_name = ( + qualified_name.rsplit(self.separator, 1)[-1] + if self.separator in qualified_name + else qualified_name ) - flat_captures.sort(key=lambda x: x[0]) - - module_name = None - current_import = None - pairs = [] - - for _, tag, text in flat_captures: - if tag == "module_name": - module_name = text - elif tag == "import_name": - if current_import is not None: - pairs.append((current_import, None)) - current_import = text - elif tag == "alias": - pairs.append((current_import, text)) - current_import = None - - if current_import is not None: - pairs.append((current_import, None)) - - return module_name, pairs - + defined = qualified_name in self.definitions + fingerprint_hit = bool(fingerprint and fingerprint in self.fingerprints) -def collect_imports(root_node, language: str) -> dict[str, str | list[str]]: - """ - Extract import statements from a Tree-sitter AST and map every - local alias to its absolute imported path. - """ - config = get_imports_language_config(language) - separator = config["separator"] - wildcard_sym = config.get("wildcard_symbol") - - query = get_query(language, "imports") - if not query or not root_node: - return {} - - import_map: dict[str, str | list[str]] = {} - wildcard_modules: list[str] = [] - - for _pattern_index, captures in query.matches(root_node): - module_name, pairs = extract_imports(captures) + imported = ( + qualified_name in self.imports + or qualified_name in self.imports.values() + ) - for imp_name, alias in pairs: - if not imp_name: - continue + callers = set() + callers.update(self.callers_of.get(short_name, set())) + callers.update(self.callers_of.get(qualified_name, set())) + called = bool(callers) - if wildcard_sym is not None and imp_name == wildcard_sym: - if module_name: - wildcard_modules.append(module_name) + if not (defined or fingerprint_hit or called or imported): continue - local_name = alias or imp_name - - if not alias and separator in imp_name: - local_name = imp_name.split(separator)[0] - - absolute_path = ( - f"{module_name}{separator}{imp_name}" if module_name else imp_name + entry = matched.setdefault( + qualified_name, + { + "symbol_name": qualified_name, + "called": False, + "defined": False, + "imported": False, + "fingerprint": None, + "reachable_from": [], + }, ) - import_map[local_name] = absolute_path - if wildcard_modules: - import_map["*"] = wildcard_modules + if defined: + entry["defined"] = True + if imported: + entry["imported"] = True + if called: + entry["called"] = True + entry["reachable_from"] = sorted(callers) + + if fingerprint_hit: + entry["fingerprint"] = fingerprint - return import_map + return matched diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 1cc142057a..855b481fed 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -22,6 +22,7 @@ import hashlib import importlib +from abc import ABC from functools import cache from django.db.models import Q @@ -192,44 +193,6 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): "pygments": symbols_pygments.get_pygments_symbols, } -TS_QUERIES = { - "Python": { - "functions": """ - (function_definition name: (identifier) @name) @function - """, - "classes": """ - (class_definition name: (identifier) @name) @class - """, - "calls": """ - (call function: (identifier) @callee) - (call function: (attribute - object: (_) @receiver - attribute: (identifier) @callee)) - """, - "imports": """ - (import_statement name: [ - (dotted_name) @import_name - (aliased_import name: (dotted_name) - @import_name alias: (identifier) @alias) - ]) - - ; Combine explicit, relative, aliased, and wildcard from_imports - (import_from_statement - module_name: [ - (dotted_name) @module_name - (relative_import) @module_name - ] - [ - name: (dotted_name) @import_name - name: (aliased_import name: (dotted_name) - @import_name alias: (identifier) @alias) - (wildcard_import) @import_name - ] - ) - """, - }, -} - @cache def load_language(language: str) -> Language: @@ -246,101 +209,265 @@ def load_language(language: str) -> Language: return Language(grammar.language()) -@cache -def get_query(language: str, kind: str) -> Query | None: - source = TS_QUERIES.get(language, {}).get(kind, "").strip() - if not source: +def create_sha256_fingerprint(text): + if not text: return None - return Query(load_language(language), source) - -def parse_code_to_ast(code_text: str, language: str): - if not code_text or not language or language not in TS_LANGUAGE_WHEELS: - return None, None - - ts_language = load_language(language) - parser = Parser(language=ts_language) - return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[language] - - -def run_query(query: Query, root_node): - """Yield ``(definition_node, name)`` pairs for function/class queries.""" - if query is None: - return + text = text.encode("utf-8", errors="replace") + return hashlib.sha256(text).hexdigest() - for _pattern_index, captures in query.matches(root_node): - def_nodes = captures.get("function") or captures.get("class") or [] - if not def_nodes: - continue - name_nodes = captures.get("name") or [] - name = ( - name_nodes[0].text.decode("utf-8", errors="replace") if name_nodes else None - ) - yield def_nodes[0], name +class LanguageQuery(ABC): + language_name: str = "" + functions_query: str = "" + classes_query: str = "" + calls_query: str = "" + imports_query: str = "" + syntax_config: dict = { + "self_keyword": None, + "separator": ".", + "wildcard_symbol": None, + } + + def __init__(self): + self.ts_language = load_language(self.language_name) + self._compiled_queries = {} + + for kind in ("functions", "classes", "calls", "imports"): + source = getattr(self, f"{kind}_query", "").strip() + self._compiled_queries[kind] = ( + Query(self.ts_language, source) if source else None + ) + def parse_code_to_ast(self, code_text: str): + if not code_text: + return None, None + parser = Parser(language=self.ts_language) + return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[ + self.language_name + ] + + def run_query(self, kind: str, root_node): + query = self._compiled_queries.get(kind) + return query.matches(root_node) if query else [] + + def get_functions(self, root_node): + for _, captures in self.run_query("functions", root_node): + def_nodes = captures.get("function") + if not def_nodes: + continue + name_nodes = captures.get("name") + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") + if name_nodes + else None + ) + yield def_nodes[0], name + + def get_classes(self, root_node): + for _, captures in self.run_query("classes", root_node): + def_nodes = captures.get("class") + if not def_nodes: + continue + name_nodes = captures.get("name") + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") + if name_nodes + else None + ) + yield def_nodes[0], name + + def get_calls(self, node): + """Yield raw (receiver_node, callee_node).""" + seen_callees = set() + for _, captures in self.run_query("calls", node): + for callee_node in captures.get("callee", []): + if callee_node.id in seen_callees: + continue + seen_callees.add(callee_node.id) + + receiver_nodes = captures.get("receiver") + receiver_node = receiver_nodes[0] if receiver_nodes else None + yield receiver_node, callee_node + + def get_imports(self, root_node): + """Yield raw (module_name, [(import_name, alias), ...]).""" + for _, captures in self.run_query("imports", root_node): + flat_captures = [] + for tag, nodes in captures.items(): + for n in nodes: + text = n.text.decode("utf-8", errors="replace").strip("'\"") + flat_captures.append((n.start_byte, tag, text)) + + flat_captures.sort(key=lambda x: x[0]) + module_name, current_import = None, None + pairs = [] + + for _, tag, text in flat_captures: + if tag == "module_name": + module_name = text + elif tag == "import_name": + if current_import is not None: + pairs.append((current_import, None)) + current_import = text + elif tag == "alias": + pairs.append((current_import, text)) + current_import = None + + if current_import is not None: + pairs.append((current_import, None)) + + yield module_name, pairs + + +class PythonTreeSitterQuery(LanguageQuery): + language_name = "Python" + functions_query = "(function_definition name: (identifier) @name) @function" + classes_query = "(class_definition name: (identifier) @name) @class" + calls_query = """ + (call function: (identifier) @callee) + (call function: (attribute object: (_) @receiver + attribute: (identifier) @callee)) + """ + imports_query = """ + (import_statement name: (dotted_name) @import_name) + (import_statement name: (aliased_import + name: (dotted_name) @import_name + alias: (identifier) @alias)) + + (import_from_statement + module_name: [(dotted_name) (relative_import)] @module_name + name: [ + (dotted_name) @import_name + (aliased_import name: (dotted_name) @import_name + alias: (identifier) @alias) + ]) + + (import_from_statement + module_name: [(dotted_name) (relative_import)] @module_name + (wildcard_import) @import_name) + """ + syntax_config = {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"} -def _root_of(node): - while node.parent is not None: - node = node.parent - return node +TS_QUERIES = { + "Python": PythonTreeSitterQuery, +} -def collect_definitions(root_node, language: str): - index: dict[int, dict] = {} - for kind in ("functions", "classes"): - query = get_query(language, kind) - for node, name in run_query(query, root_node): - index[node.id] = {"node": node, "name": name, "kind": kind} - return index +class SymbolExtractor: + def __init__(self, lang_query: LanguageQuery, root_node): + self.lang_query = lang_query + self.root_node = root_node + self.syntax_config = lang_query.syntax_config + + def _build_qualified_name(self, node, index: dict) -> str: + """ + Build fully qualified names internally + (e.g., ClassName.function_name or ClassName::function_name). + """ + parts = [] + curr = node + while curr is not None: + definition = index.get(curr.id) + if definition is not None and definition["name"]: + parts.append(definition["name"]) + curr = curr.parent + + separator = self.syntax_config.get("separator", ".") + return separator.join(reversed(parts)) + + def extract_definitions_index(self): + """Build the index of definitions with fully qualified names.""" + index: dict[int, dict] = {} + + for node, name in self.lang_query.get_functions(self.root_node): + index[node.id] = {"node": node, "name": name, "kind": "functions"} + + for node, name in self.lang_query.get_classes(self.root_node): + index[node.id] = {"node": node, "name": name, "kind": "classes"} + + for def_info in index.values(): + def_info["qualified_name"] = self._build_qualified_name( + def_info["node"], index + ) -def extract_definitions(tree, language: str, kinds=("functions", "classes")): - if tree is None: - return [] - index = collect_definitions(tree.root_node, language) - return [d["node"] for d in index.values() if d["kind"] in kinds] + return index + + def extract_changed_symbols(self, changed_lines: list[int]): + """Map changed line numbers to their enclosing symbol nodes.""" + if self.root_node is None or not changed_lines: + return [] + + definition_ids = set(self.extract_definitions_index().keys()) + if not definition_ids: + return [] + + seen = set() + enclosing = [] + + for line in changed_lines: + row = max(0, line - 1) + node = self.root_node.descendant_for_point_range((row, 0), (row, 0)) + + while node is not None: + if node.id in definition_ids and node.id not in seen: + seen.add(node.id) + enclosing.append(node) + break + node = node.parent + + return enclosing + + def extract_calls(self, node): + """Extract direct calls into (receiver_name, callee_name) text format.""" + calls = [] + for receiver_node, callee_node in self.lang_query.get_calls(node): + receiver_name = ( + receiver_node.text.decode("utf-8", errors="replace") + if receiver_node + else None + ) + callee_name = callee_node.text.decode("utf-8", errors="replace") + if callee_name: + calls.append((receiver_name, callee_name)) -def extract_symbols(tree, changed_lines: list[int], language: str): - if tree is None or not changed_lines: - return [] + return calls - definition_ids = set(collect_definitions(tree.root_node, language).keys()) - if not definition_ids: - return [] + def extract_imports(self): + """Map every local alias to its absolute imported path.""" + separator = self.syntax_config.get("separator", ".") + wildcard_sym = self.syntax_config.get("wildcard_symbol") - seen = set() - enclosing = [] + import_map: dict[str, str | list[str]] = {} + wildcard_modules: list[str] = [] - for line in changed_lines: - row = max(0, line - 1) - node = tree.root_node.descendant_for_point_range((row, 0), (row, 0)) + for module_name, pairs in self.lang_query.get_imports(self.root_node): + for imp_name, alias in pairs: + if not imp_name: + continue - while node is not None: - if node.id in definition_ids and node.id not in seen: - seen.add(node.id) - enclosing.append(node) - break - node = node.parent + if wildcard_sym is not None and imp_name == wildcard_sym: + if module_name: + wildcard_modules.append(module_name) + continue - return enclosing + local_name = alias or imp_name + if not alias and separator in imp_name: + local_name = imp_name.split(separator)[0] + absolute_path = ( + f"{module_name}{separator}{imp_name}" if module_name else imp_name + ) + import_map[local_name] = absolute_path -def qualified_name_from_index(node, index): - parts = [] - curr = node - while curr is not None: - definition = index.get(curr.id) - if definition is not None and definition["name"]: - parts.append(definition["name"]) - curr = curr.parent - return ".".join(reversed(parts)) + if wildcard_modules: + import_map["*"] = wildcard_modules + return import_map -def create_sha256_fingerprint(text): - if text is None: - return None - text = text.encode("utf-8", errors="replace") - return hashlib.sha256(text).hexdigest() +def is_supported_language(language): + """Return True if the language is supported by tree-sitter queries.""" + return bool(language) and language in TS_QUERIES diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 572dd782e6..eeeb0560a3 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -21,6 +21,7 @@ # Visit https://github.com/nexB/scancode.io for support and download. from pathlib import Path +from unittest.mock import MagicMock from unittest.mock import PropertyMock from unittest.mock import patch @@ -28,21 +29,12 @@ from scanpipe.models import Project from scanpipe.pipes import collect_and_create_codebase_resources +from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus -from scanpipe.pipes.reachability import analyze_patched_file -from scanpipe.pipes.reachability import build_symbol_metadata from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import collect_imports -from scanpipe.pipes.reachability import compute_reachable_symbols -from scanpipe.pipes.reachability import diff_changed_symbols -from scanpipe.pipes.reachability import extract_direct_calls -from scanpipe.pipes.reachability import get_symbol_reachability_results -from scanpipe.pipes.reachability import parse_diff_lines -from scanpipe.pipes.symbols import collect_definitions -from scanpipe.pipes.symbols import extract_definitions -from scanpipe.pipes.symbols import extract_symbols -from scanpipe.pipes.symbols import parse_code_to_ast -from scanpipe.pipes.symbols import qualified_name_from_index +from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.symbols import TS_QUERIES +from scanpipe.pipes.symbols import SymbolExtractor class SymbolReachabilityPipesTest(TestCase): @@ -52,24 +44,26 @@ def setUp(self): self.project1 = Project.objects.create(name="Analysis") self.project1.codebase_path.mkdir(parents=True, exist_ok=True) - @patch("scanpipe.pipes.reachability.clone_repo") - @patch("scanpipe.pipes.reachability.collect_patch_symbols") + @patch("scanpipe.pipes.reachability.GitRepositoryContext") + @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, - mock_clone_repo, + mock_git_context, ): app_text = (self.data / "app.py").read_text() vuln_text = (self.data / "vuln-app.py").read_text() fixed_text = (self.data / "fixed-app.py").read_text() diff_text = (self.data / "diff-app.patch").read_text() - diff_line_map = parse_diff_lines(diff_text=diff_text) + + analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") + diff_line_map = analyzer.parse_diff_lines(diff_text=diff_text) file_path = "app.py" removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) - vuln_meta, fixed_meta, lang = analyze_patched_file( + vuln_meta, fixed_meta, lang = analyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, removed_lines=removed_lines, @@ -90,7 +84,7 @@ def test_get_symbol_reachability_results( } ] - mock_clone_repo.return_value = str(self.project1.codebase_path) + mock_git_context.return_value.__enter__.return_value.repo = MagicMock() mock_collect_symbols.return_value = { lang: { "vulnerable": { @@ -110,48 +104,52 @@ def test_get_symbol_reachability_results( resource.programming_language = lang resource.save() - with patch("scanpipe.pipes.reachability.Repo"): - get_symbol_reachability_results(self.project1) + collect_and_store_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") self.assertEqual( results, - { - "symbols_reachability": { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "evidence": [ - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c00679511833" - "553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", - "reachable_from": [], - }, - { - "called": True, - "defined": True, - "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364" - "e558140239913bfabcc5aa77156460c2eb0a355", - "symbol_name": "serve_report.build_file_path", - "reachable_from": ["serve_report"], + [ + { + "symbols_reachability": { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", }, - ], - "fixed_symbols": ["serve_report", "serve_report.build_file_path"], - "vulnerable_symbols": [ - "serve_report", - "serve_report.build_file_path", - ], - "reachability_status": "REACHABLE", + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c00679511833" + "553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": "762e4f7d03b1bf4359c3ca364" + "e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": ["serve_report"], + }, + ], + "fixed_symbols": [ + "serve_report", + "serve_report.build_file_path", + ], + "vulnerable_symbols": [ + "serve_report", + "serve_report.build_file_path", + ], + "reachability_status": "REACHABLE", + } } - }, + ], ) def test_extract_definitions(self): @@ -169,27 +167,32 @@ def calculate_discount(price): class InventoryItem: pass """ - tree, _ = parse_code_to_ast(source_code, "Python") - functions = extract_definitions(tree, "Python", kinds=("functions",)) + + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + functions = list(lang_query.get_functions(tree.root_node)) + self.assertEqual( len(functions), 3 ) # '__init__', 'process_payment', and 'calculate_discount' - self.assertEqual(functions[0].type, "function_definition") - first_func_text = functions[0].text.decode("utf-8") + self.assertEqual(functions[0][0].type, "function_definition") + first_func_text = functions[0][0].text.decode("utf-8") self.assertIn("def __init__", first_func_text) - classes = extract_definitions(tree, "Python", kinds=("classes",)) + classes = list(lang_query.get_classes(tree.root_node)) self.assertEqual(len(classes), 2) - second_class_text = classes[1].text.decode("utf-8") + second_class_text = classes[1][0].text.decode("utf-8") self.assertIn("class InventoryItem", second_class_text) def test_extract_definitions_empty(self): - tree, _ = parse_code_to_ast("", "Python") - self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) - self.assertEqual(extract_definitions(tree, "Python", kinds=("functions",)), []) - self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) - self.assertEqual(extract_definitions(None, "Python", kinds=("classes",)), []) + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast("") + + self.assertIsNone(tree) + + tree_none, _ = lang_query.parse_code_to_ast(None) + self.assertIsNone(tree_none) def test_get_qualified_name_functions(self): source_code = """ @@ -202,14 +205,17 @@ def global_utility(): pass """ - tree, _ = parse_code_to_ast(source_code, "Python") - index = collect_definitions(tree.root_node, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) + + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + index = extractor.extract_definitions_index() - functions = extract_definitions(tree, "Python", kinds=("functions",)) + functions = list(lang_query.get_functions(tree.root_node)) self.assertEqual(len(functions), 2) - outer_function_name = qualified_name_from_index(functions[0], index) - inner_function_name = qualified_name_from_index(functions[1], index) + outer_function_name = extractor._build_qualified_name(functions[0][0], index) + inner_function_name = extractor._build_qualified_name(functions[1][0], index) self.assertEqual(outer_function_name, "CoreService.Validator.validate_payload") self.assertEqual(inner_function_name, "global_utility") @@ -220,14 +226,17 @@ class FleetManagement: class DroneController: pass """ - tree, _ = parse_code_to_ast(source_code, "Python") - index = collect_definitions(tree.root_node, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) + + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + index = extractor.extract_definitions_index() - classes = extract_definitions(tree, "Python", kinds=("classes",)) + classes = list(lang_query.get_classes(tree.root_node)) self.assertEqual(len(classes), 2) - outer_class_name = qualified_name_from_index(classes[0], index) - inner_class_name = qualified_name_from_index(classes[1], index) + outer_class_name = extractor._build_qualified_name(classes[0][0], index) + inner_class_name = extractor._build_qualified_name(classes[1][0], index) self.assertEqual(outer_class_name, "FleetManagement") self.assertEqual(inner_class_name, "FleetManagement.DroneController") @@ -235,6 +244,9 @@ class DroneController: def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) + self.assertEqual( + classify_reachability({"evidence": {}}), ReachabilityStatus.NOT_REACHABLE + ) self.assertEqual( classify_reachability({"evidence": {"fingerprint": "hash123"}}), ReachabilityStatus.REACHABLE, @@ -265,21 +277,40 @@ def inner_helper(): def process_data(payload): return payload """ - tree, _ = parse_code_to_ast(source_code, "Python") - nodes = extract_definitions(tree, "Python", kinds=("functions",)) - - metadata = build_symbol_metadata(nodes, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + index = extractor.extract_definitions_index() + vuln_nodes = extractor.extract_changed_symbols( + changed_lines=[1, 2, 3, 4, 5, 6, 7, 8, 9] + ) + metadata = PatchAnalyzer.build_symbol_metadata( + nodes=vuln_nodes, extractor=extractor, index=index + ) self.assertEqual( metadata, { + "Controller": { + "qualified_name": "Controller", + "text": "class Controller:\n" + " def process_data(payload):\n" + " def inner_helper():\n " + " return True\n " + " return payload.strip()", + "fingerprint": "de81abd637e27302d8e19c41eab8f4" + "fb6b8abdd9fc4f1fb31d354bc7b23f6d4d", + "start_line": 2, + "end_line": 6, + "node_type": "class_definition", + }, "Controller.process_data": { "qualified_name": "Controller.process_data", "text": "def process_data(payload):\n" " def inner_helper():\n" - " return True\n " - " return payload.strip()", - "fingerprint": "b0d0ad9a92209a6d79b84e932ce302" - "a8bc9054a405131adf7dc21e06e2e7c0c1", + " return True\n" + " return payload.strip()", + "fingerprint": "b0d0ad9a92209a6d79b84e932ce3" + "02a8bc9054a405131adf7dc21e06e2e7c0c1", "start_line": 3, "end_line": 6, "node_type": "function_definition", @@ -287,21 +318,12 @@ def process_data(payload): "Controller.process_data.inner_helper": { "qualified_name": "Controller.process_data.inner_helper", "text": "def inner_helper():\n return True", - "fingerprint": "ee2e246e01e960826cb39a9466e5" - "8095d209fdd1cbf8458630be430b3371d6a3", + "fingerprint": "ee2e246e01e960826cb39a9466e58095" + "d209fdd1cbf8458630be430b3371d6a3", "start_line": 4, "end_line": 5, "node_type": "function_definition", }, - "process_data": { - "qualified_name": "process_data", - "text": "def process_data(payload):\n return payload", - "fingerprint": "9b2797712c9ab60ea8452a4413965" - "c94d1b2f63739cab7de695e7b1dc0cf439a", - "start_line": 9, - "end_line": 10, - "node_type": "function_definition", - }, }, ) @@ -339,8 +361,9 @@ def test_diff_changed_symbols(self): }, } - vuln_only, fixed_only = diff_changed_symbols(vuln_meta, fixed_meta) - + vuln_only, fixed_only = PatchAnalyzer.diff_changed_symbols( + vuln_meta, fixed_meta + ) self.assertEqual( vuln_only, { @@ -374,11 +397,12 @@ def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") - diff_line_map = parse_diff_lines(diff_text=diff_text) + + diff_line_map = PatchAnalyzer.parse_diff_lines(diff_text=diff_text) file_path = "app.py" removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) - vuln_meta, fixed_meta, lang = analyze_patched_file( + vuln_meta, fixed_meta, lang = PatchAnalyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, removed_lines=removed_lines, @@ -491,13 +515,15 @@ def test_extract_symbols(self): " return build_path(request)\n" # Line 5 (Row 4) ) - tree, _ = parse_code_to_ast(source_code, "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) changed_lines = [4] - enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + changed_symbols = extractor.extract_changed_symbols(changed_lines) - self.assertEqual(len(enclosing_symbols), 1) - target_node = enclosing_symbols[0] + self.assertEqual(len(changed_symbols), 1) + target_node = changed_symbols[0] self.assertEqual(target_node.type, "function_definition") node_text = target_node.text.decode("utf-8") @@ -511,29 +537,17 @@ def test_extract_symbols_deduplication(self): " return price + amount\n" # Line 3 -> Changed ) - tree, _ = parse_code_to_ast(source_code, "Python") - changed_lines = [2, 3] + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(source_code) - enclosing_symbols = extract_symbols(tree, changed_lines, "Python") + changed_lines = [2, 3] + symbol_extractor = SymbolExtractor( + lang_query=lang_query, root_node=tree.root_node + ) + enclosing_symbols = symbol_extractor.extract_changed_symbols(changed_lines) self.assertEqual(len(enclosing_symbols), 1) self.assertEqual(enclosing_symbols[0].type, "function_definition") - def test_compute_reachable_symbols(self): - edges_qualified = { - "app.main": {"app.helper", "app.safe_func"}, - "app.helper": {"app.vuln_func"}, - "app.direct_caller": {"app.vuln_func"}, - "app.unrelated": {"app.safe_func"}, - } - - target_qns = ["app.vuln_func"] - reachable, has_direct = compute_reachable_symbols(edges_qualified, target_qns) - - self.assertTrue(has_direct) - - expected_reachable = {"app.main", "app.helper", "app.direct_caller"} - self.assertEqual(reachable, expected_reachable) - def test_collect_imports(self): source_code = """ from django.db import models @@ -545,9 +559,10 @@ def test_collect_imports(self): from math import * """.strip() - tree, _ = parse_code_to_ast(source_code, "Python") - real_root_node = tree.root_node - result = collect_imports(real_root_node, language="Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + result = extractor.extract_imports() expected_map = { "models": "django.db.models", @@ -571,11 +586,30 @@ def clean_function(): return hello() + x + y """.strip() - tree, _ = parse_code_to_ast(source_code, "Python") - functions = extract_definitions(tree, "Python", kinds=("functions",)) - - result = extract_direct_calls(functions[1], "Python") + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + result = extractor.extract_calls(node=tree.root_node) self.assertEqual( result, [(None, "hello")], ) + + def test_extract_direct_calls(self): + python_source = """ +self.update() +process_data() +user.save() + """.strip() + + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=python_source) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + python_calls = extractor.extract_calls(node=tree.root_node) + + expected_python = [ + ("self", "update"), + (None, "process_data"), + ("user", "save"), + ] + self.assertEqual(expected_python, python_calls) From c4ac68ac91594d6f71d94233e590ca57e90e3396 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 16:03:05 +0300 Subject: [PATCH 12/50] Add support for getting constant symbols Fix formating tests Signed-off-by: ziad hany --- scanpipe/pipes/symbols.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 855b481fed..94c8d5c936 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -219,6 +219,7 @@ def create_sha256_fingerprint(text): class LanguageQuery(ABC): language_name: str = "" + constants_query: str = "" functions_query: str = "" classes_query: str = "" calls_query: str = "" @@ -233,7 +234,7 @@ def __init__(self): self.ts_language = load_language(self.language_name) self._compiled_queries = {} - for kind in ("functions", "classes", "calls", "imports"): + for kind in ("constants", "functions", "classes", "calls", "imports"): source = getattr(self, f"{kind}_query", "").strip() self._compiled_queries[kind] = ( Query(self.ts_language, source) if source else None @@ -319,9 +320,30 @@ def get_imports(self, root_node): yield module_name, pairs + def get_constants(self, root_node): + """Yield raw (constant_node, name).""" + for _, captures in self.run_query("constants", root_node): + def_nodes = captures.get("constant") + if not def_nodes: + continue + name_nodes = captures.get("name") + name = ( + name_nodes[0].text.decode("utf-8", errors="replace") + if name_nodes + else None + ) + yield def_nodes[0], name + class PythonTreeSitterQuery(LanguageQuery): language_name = "Python" + constants_query = """ + (assignment + left: (identifier) @name) @constant + + (assignment + left: (pattern_list (identifier) @name)) @constant + """ functions_query = "(function_definition name: (identifier) @name) @function" classes_query = "(class_definition name: (identifier) @name) @class" calls_query = """ @@ -334,7 +356,6 @@ class PythonTreeSitterQuery(LanguageQuery): (import_statement name: (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias)) - (import_from_statement module_name: [(dotted_name) (relative_import)] @module_name name: [ @@ -342,7 +363,6 @@ class PythonTreeSitterQuery(LanguageQuery): (aliased_import name: (dotted_name) @import_name alias: (identifier) @alias) ]) - (import_from_statement module_name: [(dotted_name) (relative_import)] @module_name (wildcard_import) @import_name) @@ -387,6 +407,9 @@ def extract_definitions_index(self): for node, name in self.lang_query.get_classes(self.root_node): index[node.id] = {"node": node, "name": name, "kind": "classes"} + for node, name in self.lang_query.get_constants(self.root_node): + index[node.id] = {"node": node, "name": name, "kind": "constants"} + for def_info in index.values(): def_info["qualified_name"] = self._build_qualified_name( def_info["node"], index From d48c18e3d843d318138b30e8d45d4777b1879a5e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 16:52:17 +0300 Subject: [PATCH 13/50] Remove dead code and fix the test Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 15 ++++++--------- scanpipe/pipes/symbols.py | 29 ++++++++++------------------- 2 files changed, 16 insertions(+), 28 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index b6bf86c837..3a51372332 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -74,9 +74,8 @@ def detect_language_with_scancode(file_path, content): class GitRepositoryContext: - def __init__(self, vcs_url: str, commit_hash: str = None): + def __init__(self, vcs_url: str): self.vcs_url = vcs_url - self.commit_hash = commit_hash self.repo_path = None self._repo = None @@ -84,13 +83,11 @@ def __enter__(self) -> "GitRepositoryContext": self.repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") try: self._repo = Repo.clone_from(self.vcs_url, self.repo_path) - if self.commit_hash: - self._repo.git.checkout(self.commit_hash) return self except Exception as exc: self._cleanup() raise ValueError( - f"Failed to clone/checkout {self.vcs_url}@{self.commit_hash}" + f"Failed to clone/checkout {self.vcs_url}" ) from exc def __exit__(self, exc_type, exc_val, exc_tb): @@ -288,7 +285,7 @@ def build_symbol_metadata( @classmethod def analyze( - self, vulnerable_text, fixed_text, removed_lines, added_lines, file_path + cls, vulnerable_text, fixed_text, removed_lines, added_lines, file_path ): vulnerable_text = normalize_text(vulnerable_text) fixed_text = normalize_text(fixed_text) @@ -326,7 +323,7 @@ def analyze( vuln_nodes = vuln_extractor.extract_changed_symbols( changed_lines=removed_lines ) - vuln_meta_all = self.build_symbol_metadata( + vuln_meta_all = cls.build_symbol_metadata( nodes=vuln_nodes, extractor=vuln_extractor ) @@ -337,11 +334,11 @@ def analyze( fixed_nodes = fixed_extractor.extract_changed_symbols( changed_lines=added_lines ) - fixed_meta_all = self.build_symbol_metadata( + fixed_meta_all = cls.build_symbol_metadata( fixed_nodes, extractor=fixed_extractor ) - vuln_meta, fixed_meta = self.diff_changed_symbols( + vuln_meta, fixed_meta = cls.diff_changed_symbols( vuln_meta=vuln_meta_all, fixed_meta=fixed_meta_all ) return vuln_meta, fixed_meta, language diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 94c8d5c936..adf50980ee 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -26,16 +26,6 @@ from functools import cache from django.db.models import Q - -from source_inspector import symbols_ctags -from source_inspector import symbols_pygments -from source_inspector import symbols_tree_sitter -from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS -from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled -from tree_sitter import Language -from tree_sitter import Parser -from tree_sitter import Query - from aboutcode.pipeline import LoopProgress @@ -186,16 +176,12 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): } ) - -SYMBOLS_TYPE_SUPPORTED = { - "ctags": symbols_ctags.get_symbols, - "tree_sitter": symbols_tree_sitter.get_treesitter_symbols, - "pygments": symbols_pygments.get_pygments_symbols, -} - - @cache -def load_language(language: str) -> Language: +def load_language(language: str): + from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS + from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled + from tree_sitter import Language + if language not in TS_LANGUAGE_WHEELS: raise ValueError(f"Unsupported language: {language}") @@ -231,6 +217,8 @@ class LanguageQuery(ABC): } def __init__(self): + from tree_sitter import Query + self.ts_language = load_language(self.language_name) self._compiled_queries = {} @@ -241,6 +229,9 @@ def __init__(self): ) def parse_code_to_ast(self, code_text: str): + from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS + from tree_sitter import Parser + if not code_text: return None, None parser = Parser(language=self.ts_language) From 66864870a40c0a5960fa134c2a26ff646b51a464 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 17:13:44 +0300 Subject: [PATCH 14/50] Add unidiff to uv.lock Fix formating Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 4 +--- scanpipe/pipes/symbols.py | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 3a51372332..635941da20 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -86,9 +86,7 @@ def __enter__(self) -> "GitRepositoryContext": return self except Exception as exc: self._cleanup() - raise ValueError( - f"Failed to clone/checkout {self.vcs_url}" - ) from exc + raise ValueError(f"Failed to clone/checkout {self.vcs_url}") from exc def __exit__(self, exc_type, exc_val, exc_tb): self._cleanup() diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index adf50980ee..d5ed4bf5b9 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -26,6 +26,7 @@ from functools import cache from django.db.models import Q + from aboutcode.pipeline import LoopProgress @@ -176,6 +177,7 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): } ) + @cache def load_language(language: str): from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS From a9076d37b1a4ddfc60eb01dd4e43a1bba8aca1d1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 18:11:32 +0300 Subject: [PATCH 15/50] Add support constant to resource analyzer Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 59 ++++++++++++++++++++-------------- scanpipe/pipes/symbols.py | 16 ++++----- 2 files changed, 42 insertions(+), 33 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 635941da20..0e1f7f59b5 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -463,16 +463,32 @@ def classify_reachability(evidence): class ResourceAnalyzer: def __init__(self, resource_text: str, language: str): - self.resource_text = resource_text + self.resource_text = normalize_text(resource_text) self.language = language + def process_node( + self, node, extractor, definitions_index, definitions: set, fingerprints: set + ) -> str | None: + """Extract the qualified name, update definitions, and fingerprint""" + qualified_name = extractor._build_qualified_name(node, definitions_index) + if not qualified_name: + return None + + definitions.add(qualified_name) + body_text = node.text.decode("utf-8", errors="replace") + fingerprint = create_sha256_fingerprint(body_text) + + if fingerprint: + fingerprints.add(fingerprint) + + return qualified_name + def build_index(self) -> dict | None: - text = normalize_text(self.resource_text) - if not is_supported_language(self.language) or not text: + if not is_supported_language(self.language) or not self.resource_text: return None lang_query = TS_QUERIES[self.language]() - tree, _ = lang_query.parse_code_to_ast(text) + tree, _ = lang_query.parse_code_to_ast(self.resource_text) if tree is None: return None @@ -487,29 +503,24 @@ def build_index(self) -> dict | None: callers_of = {} # callee_name -> set of caller_qualified_names for node, _ in lang_query.get_functions(tree.root_node): - qualified_name = extractor._build_qualified_name(node, definitions_index) - if not qualified_name: - continue - - definitions.add(qualified_name) - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_sha256_fingerprint(body_text) - if fingerprint: - fingerprints.add(fingerprint) - - for _, callee_name in extractor.extract_calls(node): - callers_of.setdefault(callee_name, set()).add(qualified_name) + qualified_name = self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) + if qualified_name: + for _, callee_name in extractor.extract_calls(node): + callers_of.setdefault(callee_name, set()).add(qualified_name) for node, _ in lang_query.get_classes(tree.root_node): - qualified_name = extractor._build_qualified_name(node, definitions_index) - if not qualified_name: - continue + self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) - definitions.add(qualified_name) - body_text = node.text.decode("utf-8", errors="replace") - fingerprint = create_sha256_fingerprint(body_text) - if fingerprint: - fingerprints.add(fingerprint) + for node, _ in lang_query.get_constants(tree.root_node): + # Skip constants defined inside functions, classes, or other blocks + if node.parent == tree.root_node: + self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) return { "definitions": definitions, diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index d5ed4bf5b9..b32d8bd955 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -180,21 +180,19 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): @cache def load_language(language: str): - from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS - from source_inspector.symbols_tree_sitter import TreeSitterWheelNotInstalled - from tree_sitter import Language + from source_inspector import symbols_tree_sitter - if language not in TS_LANGUAGE_WHEELS: + if language not in symbols_tree_sitter.TS_LANGUAGE_WHEELS: raise ValueError(f"Unsupported language: {language}") - wheel = TS_LANGUAGE_WHEELS[language]["wheel"] + wheel = symbols_tree_sitter.TS_LANGUAGE_WHEELS[language]["wheel"] try: grammar = importlib.import_module(wheel) except ModuleNotFoundError as exc: - raise TreeSitterWheelNotInstalled( + raise symbols_tree_sitter.TreeSitterWheelNotInstalled( f"Grammar wheel '{wheel}' is not installed." ) from exc - return Language(grammar.language()) + return symbols_tree_sitter.Language(grammar.language()) def create_sha256_fingerprint(text): @@ -231,13 +229,13 @@ def __init__(self): ) def parse_code_to_ast(self, code_text: str): - from source_inspector.symbols_tree_sitter import TS_LANGUAGE_WHEELS + from source_inspector import symbols_tree_sitter from tree_sitter import Parser if not code_text: return None, None parser = Parser(language=self.ts_language) - return parser.parse(code_text.encode("utf-8")), TS_LANGUAGE_WHEELS[ + return parser.parse(code_text.encode("utf-8")), symbols_tree_sitter.TS_LANGUAGE_WHEELS[ self.language_name ] From 28676436397046584c68f06eb635e52f27b354bd Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 1 Jul 2026 18:16:48 +0300 Subject: [PATCH 16/50] Fix formating error in CI Signed-off-by: ziad hany --- scanpipe/pipes/symbols.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index b32d8bd955..9c517d3ff6 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -235,9 +235,9 @@ def parse_code_to_ast(self, code_text: str): if not code_text: return None, None parser = Parser(language=self.ts_language) - return parser.parse(code_text.encode("utf-8")), symbols_tree_sitter.TS_LANGUAGE_WHEELS[ - self.language_name - ] + return parser.parse( + code_text.encode("utf-8") + ), symbols_tree_sitter.TS_LANGUAGE_WHEELS[self.language_name] def run_query(self, kind: str, root_node): query = self._compiled_queries.get(kind) From 4b48ad72d7d131cc5563e85907871c03861d01d3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 2 Jul 2026 03:57:22 +0300 Subject: [PATCH 17/50] Add a test for constants, extract_imports,collect_imports Fix a bug in extract_imports Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 20 +++-------- scanpipe/pipes/symbols.py | 27 +++++++-------- .../tests/pipes/test_symbols_reachability.py | 34 ++++++++++++++++++- 3 files changed, 49 insertions(+), 32 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 0e1f7f59b5..d9e75f4cfa 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -19,7 +19,6 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. - import os import shutil import tempfile @@ -174,27 +173,16 @@ def parse_diff_lines(cls, diff_text): added = [] for hunk in patched_file: - hunk_removed = [ + removed.extend( line.source_line_no for line in hunk if line.is_removed and line.source_line_no - ] - hunk_added = [ + ) + added.extend( line.target_line_no for line in hunk if line.is_added and line.target_line_no - ] - - if hunk_added and not hunk_removed: - anchor = max(hunk.source_start, 1) - hunk_removed = [anchor] - - if hunk_removed and not hunk_added: - anchor = max(hunk.target_start, 1) - hunk_added = [anchor] - - removed.extend(hunk_removed) - added.extend(hunk_added) + ) candidates = { patched_file.path, diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 9c517d3ff6..0f25819d51 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -287,14 +287,12 @@ def get_imports(self, root_node): for _, captures in self.run_query("imports", root_node): flat_captures = [] for tag, nodes in captures.items(): - for n in nodes: - text = n.text.decode("utf-8", errors="replace").strip("'\"") - flat_captures.append((n.start_byte, tag, text)) + for node in nodes: + text = node.text.decode("utf-8", errors="replace").strip("'\"") + flat_captures.append((node.start_byte, tag, text)) - flat_captures.sort(key=lambda x: x[0]) module_name, current_import = None, None pairs = [] - for _, tag, text in flat_captures: if tag == "module_name": module_name = text @@ -328,13 +326,7 @@ def get_constants(self, root_node): class PythonTreeSitterQuery(LanguageQuery): language_name = "Python" - constants_query = """ - (assignment - left: (identifier) @name) @constant - - (assignment - left: (pattern_list (identifier) @name)) @constant - """ + constants_query = "(assignment left: (identifier) @name) @constant" functions_query = "(function_definition name: (identifier) @name) @function" classes_query = "(class_definition name: (identifier) @name) @class" calls_query = """ @@ -471,9 +463,14 @@ def extract_imports(self): if not alias and separator in imp_name: local_name = imp_name.split(separator)[0] - absolute_path = ( - f"{module_name}{separator}{imp_name}" if module_name else imp_name - ) + if module_name: + if module_name == separator: + absolute_path = f"{separator}{imp_name}" + else: + absolute_path = f"{module_name}{separator}{imp_name}" + else: + absolute_path = imp_name + import_map[local_name] = absolute_path if wildcard_modules: diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index eeeb0560a3..620e886c05 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -154,6 +154,7 @@ def test_get_symbol_reachability_results( def test_extract_definitions(self): source_code = """ +price = 0 class OrderManager: def __init__(self, order_id): self.order_id = order_id @@ -185,6 +186,9 @@ class InventoryItem: second_class_text = classes[1][0].text.decode("utf-8") self.assertIn("class InventoryItem", second_class_text) + constants = list(lang_query.get_constants(tree.root_node)) + self.assertEqual(len(constants), 1) + def test_extract_definitions_empty(self): lang_query = TS_QUERIES["Python"]() tree, _ = lang_query.parse_code_to_ast("") @@ -569,7 +573,7 @@ def test_collect_imports(self): "os": "os.path", "np": "numpy", "d": "a.b.c", - "utils": "..utils", + "utils": ".utils", "engine": "..core.engine", "*": ["math"], } @@ -613,3 +617,31 @@ def test_extract_direct_calls(self): ("user", "save"), ] self.assertEqual(expected_python, python_calls) + + def test_extract_imports(self): + source_code = """ +from django.db import models +import os.path +import numpy as np +from a.b import c as d +from . import utils +from ..core import engine +from math import * + """.strip() + + lang_query = TS_QUERIES["Python"]() + tree, _ = lang_query.parse_code_to_ast(code_text=source_code) + extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) + result = extractor.extract_imports() + + expected_map = { + "models": "django.db.models", + "os": "os.path", + "np": "numpy", + "d": "a.b.c", + "utils": ".utils", + "engine": "..core.engine", + "*": ["math"], + } + + self.assertEqual(result, expected_map) From 1bd85f674a8d7e20606b7ecd494689ee19dfec3c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 16 Jul 2026 02:02:04 +0300 Subject: [PATCH 18/50] Remove dependency and copy only the required file Signed-off-by: ziad hany --- scanpipe/pipes/unidiff/patch.py | 667 ++++++++++++++++++++++++ scanpipe/pipes/unidiff/patch.py.ABOUT | 13 + scanpipe/pipes/unidiff/patch.py.LICENSE | 20 + 3 files changed, 700 insertions(+) create mode 100644 scanpipe/pipes/unidiff/patch.py create mode 100644 scanpipe/pipes/unidiff/patch.py.ABOUT create mode 100644 scanpipe/pipes/unidiff/patch.py.LICENSE diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py new file mode 100644 index 0000000000..c51bdbef79 --- /dev/null +++ b/scanpipe/pipes/unidiff/patch.py @@ -0,0 +1,667 @@ +# Extracted essential patch code analyzer and modified from the original unidiff library: +# https://github.com/matiasb/python-unidiff/blob/2771a878f7bc6619e625feb4dbad3427f57f5237/unidiff/patch.py + +# -*- coding: utf-8 -*- + +# The MIT License (MIT) +# Copyright (c) 2014-2023 Matias Bordese +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +# OR OTHER DEALINGS IN THE SOFTWARE. + +from __future__ import unicode_literals +import re +from io import StringIO +from typing import Iterable, Optional, Union + +class UnidiffParseError(Exception): ... + +open_file = open +make_str = str +implements_to_string = lambda x: x +unicode = str +basestring = str + +RE_SOURCE_FILENAME = re.compile( + r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') +RE_TARGET_FILENAME = re.compile( + r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') + + +# check diff git line for git renamed files support +RE_DIFF_GIT_HEADER = re.compile( + r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)') +RE_DIFF_GIT_HEADER_URI_LIKE = re.compile( + r'^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)') +RE_DIFF_GIT_HEADER_NO_PREFIX = re.compile( + r'^diff --git (?P[^\t\n]+) (?P[^\t\n]+)') + +# check diff git new file marker `deleted file mode 100644` +RE_DIFF_GIT_DELETED_FILE = re.compile(r'^deleted file mode \d+$') + +# check diff git new file marker `new file mode 100644` +RE_DIFF_GIT_NEW_FILE = re.compile(r'^new file mode \d+$') + + +# @@ (source offset, length) (target offset, length) @@ (section header) +RE_HUNK_HEADER = re.compile( + r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") + +# kept line (context) +# \n empty line (treat like context) +# + added line +# - deleted line +# \ No newline case +RE_HUNK_BODY_LINE = re.compile( + r'^(?P[- \+\\])(?P.*)', re.DOTALL) +RE_HUNK_EMPTY_BODY_LINE = re.compile( + r'^(?P[- \+\\]?)(?P[\r\n]{1,2})', re.DOTALL) + +RE_NO_NEWLINE_MARKER = re.compile(r'^\\ No newline at end of file') + +RE_BINARY_DIFF = re.compile( + r'^Binary files? ' + r'(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?' + r'(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)') + +DEFAULT_ENCODING = 'UTF-8' + +DEV_NULL = '/dev/null' +LINE_TYPE_ADDED = '+' +LINE_TYPE_REMOVED = '-' +LINE_TYPE_CONTEXT = ' ' +LINE_TYPE_EMPTY = '' +LINE_TYPE_NO_NEWLINE = '\\' +LINE_VALUE_NO_NEWLINE = ' No newline at end of file' + +@implements_to_string +class Line(object): + """A diff line.""" + + def __init__(self, value, line_type, + source_line_no=None, target_line_no=None, diff_line_no=None): + # type: (str, str, Optional[int], Optional[int], Optional[int]) -> None + super(Line, self).__init__() + self.source_line_no = source_line_no + self.target_line_no = target_line_no + self.diff_line_no = diff_line_no + self.line_type = line_type + self.value = value + + def __repr__(self): + # type: () -> str + return make_str("") % (self.line_type, self.value) + + def __str__(self): + # type: () -> str + return "%s%s" % (self.line_type, self.value) + + def __eq__(self, other): + # type: (Line) -> bool + return (self.source_line_no == other.source_line_no and + self.target_line_no == other.target_line_no and + self.diff_line_no == other.diff_line_no and + self.line_type == other.line_type and + self.value == other.value) + + @property + def is_added(self): + # type: () -> bool + return self.line_type == LINE_TYPE_ADDED + + @property + def is_removed(self): + # type: () -> bool + return self.line_type == LINE_TYPE_REMOVED + + @property + def is_context(self): + # type: () -> bool + return self.line_type == LINE_TYPE_CONTEXT + + +@implements_to_string +class PatchInfo(list): + """Lines with extended patch info. + + Format of this info is not documented and it very much depends on + patch producer. + + """ + + def __repr__(self): + # type: () -> str + value = "" % self[0].strip() + return make_str(value) + + def __str__(self): + # type: () -> str + return ''.join(unicode(line) for line in self) + + +@implements_to_string +class Hunk(list): + """Each of the modified blocks of a file.""" + + def __init__(self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, + section_header=''): + # type: (int, int, int, int, str) -> None + super(Hunk, self).__init__() + if src_len is None: + src_len = 1 + if tgt_len is None: + tgt_len = 1 + self.source_start = int(src_start) + self.source_length = int(src_len) + self.target_start = int(tgt_start) + self.target_length = int(tgt_len) + self.section_header = section_header + self._added = None # Optional[int] + self._removed = None # Optional[int] + + def __repr__(self): + # type: () -> str + value = "" % (self.source_start, + self.source_length, + self.target_start, + self.target_length, + self.section_header) + return make_str(value) + + def __str__(self): + # type: () -> str + # section header is optional and thus we output it only if it's present + head = "@@ -%d,%d +%d,%d @@%s\n" % ( + self.source_start, self.source_length, + self.target_start, self.target_length, + ' ' + self.section_header if self.section_header else '') + content = ''.join(unicode(line) for line in self) + return head + content + + def append(self, line): + # type: (Line) -> None + """Append the line to hunk, and keep track of source/target lines.""" + # Make sure the line is encoded correctly. This is a no-op except for + # potentially raising a UnicodeDecodeError. + str(line) + super(Hunk, self).append(line) + + @property + def added(self): + # type: () -> Optional[int] + if self._added is not None: + return self._added + # re-calculate each time to allow for hunk modifications + # (which should mean metadata_only switch wasn't used) + return sum(1 for line in self if line.is_added) + + @property + def removed(self): + # type: () -> Optional[int] + if self._removed is not None: + return self._removed + # re-calculate each time to allow for hunk modifications + # (which should mean metadata_only switch wasn't used) + return sum(1 for line in self if line.is_removed) + + def is_valid(self): + # type: () -> bool + """Check hunk header data matches entered lines info.""" + return (len(self.source) == self.source_length and + len(self.target) == self.target_length) + + def source_lines(self): + # type: () -> Iterable[Line] + """Hunk lines from source file (generator).""" + return (l for l in self if l.is_context or l.is_removed) + + @property + def source(self): + # type: () -> Iterable[str] + return [str(l) for l in self.source_lines()] + + def target_lines(self): + # type: () -> Iterable[Line] + """Hunk lines from target file (generator).""" + return (l for l in self if l.is_context or l.is_added) + + @property + def target(self): + # type: () -> Iterable[str] + return [str(l) for l in self.target_lines()] + + +class PatchedFile(list): + """Patch updated file, it is a list of Hunks.""" + + def __init__(self, patch_info=None, source='', target='', + source_timestamp=None, target_timestamp=None, + is_binary_file=False): + # type: (Optional[PatchInfo], str, str, Optional[str], Optional[str], bool, bool) -> None + super(PatchedFile, self).__init__() + self.patch_info = patch_info + self.source_file = source + self.source_timestamp = source_timestamp + self.target_file = target + self.target_timestamp = target_timestamp + self.is_binary_file = is_binary_file + + def __repr__(self): + # type: () -> str + return make_str("") % make_str(self.path) + + def __str__(self): + # type: () -> str + source = '' + target = '' + # patch info is optional + info = '' if self.patch_info is None else str(self.patch_info) + if not self.is_binary_file and self: + source = "--- %s%s\n" % ( + self.source_file, + '\t' + self.source_timestamp if self.source_timestamp else '') + target = "+++ %s%s\n" % ( + self.target_file, + '\t' + self.target_timestamp if self.target_timestamp else '') + hunks = ''.join(unicode(hunk) for hunk in self) + return info + source + target + hunks + + def _parse_hunk(self, header, diff, encoding, metadata_only): + # type: (str, enumerate[str], Optional[str], bool) -> None + """Parse hunk details.""" + header_info = RE_HUNK_HEADER.match(header) + hunk_info = header_info.groups() + hunk = Hunk(*hunk_info) + + source_line_no = hunk.source_start + target_line_no = hunk.target_start + expected_source_end = source_line_no + hunk.source_length + expected_target_end = target_line_no + hunk.target_length + added = 0 + removed = 0 + + for diff_line_no, line in diff: + if encoding is not None: + line = line.decode(encoding) + + if metadata_only: + # quick line type detection, no regex required + line_type = line[0] if line else LINE_TYPE_CONTEXT + if line_type not in (LINE_TYPE_ADDED, + LINE_TYPE_REMOVED, + LINE_TYPE_CONTEXT, + LINE_TYPE_NO_NEWLINE): + raise UnidiffParseError( + 'Hunk diff line expected: %s' % line) + + if line_type == LINE_TYPE_ADDED: + target_line_no += 1 + added += 1 + elif line_type == LINE_TYPE_REMOVED: + source_line_no += 1 + removed += 1 + elif line_type == LINE_TYPE_CONTEXT: + target_line_no += 1 + source_line_no += 1 + + # no file content tracking + original_line = None + + else: + # parse diff line content + valid_line = RE_HUNK_BODY_LINE.match(line) + if not valid_line: + valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) + + if not valid_line: + raise UnidiffParseError( + 'Hunk diff line expected: %s' % line) + + line_type = valid_line.group('line_type') + if line_type == LINE_TYPE_EMPTY: + line_type = LINE_TYPE_CONTEXT + + value = valid_line.group('value') # type: str + original_line = Line(value, line_type=line_type) + + if line_type == LINE_TYPE_ADDED: + original_line.target_line_no = target_line_no + target_line_no += 1 + elif line_type == LINE_TYPE_REMOVED: + original_line.source_line_no = source_line_no + source_line_no += 1 + elif line_type == LINE_TYPE_CONTEXT: + original_line.target_line_no = target_line_no + original_line.source_line_no = source_line_no + target_line_no += 1 + source_line_no += 1 + elif line_type == LINE_TYPE_NO_NEWLINE: + pass + else: + original_line = None + + # stop parsing if we got past expected number of lines + if (source_line_no > expected_source_end or + target_line_no > expected_target_end): + raise UnidiffParseError('Hunk is longer than expected') + + if original_line: + original_line.diff_line_no = diff_line_no + hunk.append(original_line) + + # if hunk source/target lengths are ok, hunk is complete + if (source_line_no == expected_source_end and + target_line_no == expected_target_end): + break + + # report an error if we haven't got expected number of lines + if (source_line_no < expected_source_end or + target_line_no < expected_target_end): + raise UnidiffParseError('Hunk is shorter than expected') + + if metadata_only: + # HACK: set fixed calculated values when metadata_only is enabled + hunk._added = added + hunk._removed = removed + + self.append(hunk) + + def _add_no_newline_marker_to_last_hunk(self): + # type: () -> None + if not self: + raise UnidiffParseError( + 'Unexpected marker:' + LINE_VALUE_NO_NEWLINE) + last_hunk = self[-1] + last_hunk.append( + Line(LINE_VALUE_NO_NEWLINE + '\n', line_type=LINE_TYPE_NO_NEWLINE)) + + def _append_trailing_empty_line(self): + # type: () -> None + if not self: + raise UnidiffParseError('Unexpected trailing newline character') + last_hunk = self[-1] + last_hunk.append(Line('\n', line_type=LINE_TYPE_EMPTY)) + + @property + def path(self): + # type: () -> str + """Return the file path abstracted from VCS.""" + filepath = self.source_file + if filepath in (None, DEV_NULL) or ( + self.is_rename and self.target_file not in (None, DEV_NULL)): + # if this is a rename, prefer the target filename + filepath = self.target_file + + quoted = filepath.startswith('"') and filepath.endswith('"') + if quoted: + filepath = filepath[1:-1] + + if filepath.startswith('a/') or filepath.startswith('b/'): + filepath = filepath[2:] + + if quoted: + filepath = '"{}"'.format(filepath) + + return filepath + + @property + def added(self): + # type: () -> int + """Return the file total added lines.""" + return sum([hunk.added for hunk in self]) + + @property + def removed(self): + # type: () -> int + """Return the file total removed lines.""" + return sum([hunk.removed for hunk in self]) + + @property + def is_rename(self): + return (self.source_file != DEV_NULL + and self.target_file != DEV_NULL + and self.source_file[2:] != self.target_file[2:]) + + @property + def is_added_file(self): + # type: () -> bool + """Return True if this patch adds the file.""" + if self.source_file == DEV_NULL: + return True + return (len(self) == 1 and self[0].source_start == 0 and + self[0].source_length == 0) + + @property + def is_removed_file(self): + # type: () -> bool + """Return True if this patch removes the file.""" + if self.target_file == DEV_NULL: + return True + return (len(self) == 1 and self[0].target_start == 0 and + self[0].target_length == 0) + + @property + def is_modified_file(self): + # type: () -> bool + """Return True if this patch modifies the file.""" + return not (self.is_added_file or self.is_removed_file) + + +@implements_to_string +class PatchSet(list): + """A list of PatchedFiles.""" + + def __init__(self, f, encoding=None, metadata_only=False): + # type: (Union[StringIO, str], Optional[str], bool) -> None + super(PatchSet, self).__init__() + + # convert string inputs to StringIO objects + if isinstance(f, basestring): + f = self._convert_string(f, encoding) # type: StringIO + + # make sure we pass an iterator object to parse + data = iter(f) + # if encoding is None, assume we are reading unicode data + # when metadata_only is True, only perform a minimal metadata parsing + # (ie. hunks without content) which is around 2.5-6 times faster; + # it will still validate the diff metadata consistency and get counts + self._parse(data, encoding=encoding, metadata_only=metadata_only) + + def __repr__(self): + # type: () -> str + return make_str('') % super(PatchSet, self).__repr__() + + def __str__(self): + # type: () -> str + return ''.join(unicode(patched_file) for patched_file in self) + + def _parse(self, diff, encoding, metadata_only): + # type: (StringIO, Optional[str], bool) -> None + current_file = None + patch_info = None + + diff = enumerate(diff, 1) + for unused_diff_line_no, line in diff: + if encoding is not None: + line = line.decode(encoding) + + # check for a git file rename + is_diff_git_header = RE_DIFF_GIT_HEADER.match(line) or \ + RE_DIFF_GIT_HEADER_URI_LIKE.match(line) or \ + RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) + if is_diff_git_header: + patch_info = PatchInfo() + source_file = is_diff_git_header.group('source') + target_file = is_diff_git_header.group('target') + current_file = PatchedFile( + patch_info, source_file, target_file, None, None) + self.append(current_file) + patch_info.append(line) + continue + + # check for a git new file + is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) + if is_diff_git_new_file: + if current_file is None or patch_info is None: + raise UnidiffParseError('Unexpected new file found: %s' % line) + current_file.source_file = DEV_NULL + patch_info.append(line) + continue + + # check for a git deleted file + is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) + if is_diff_git_deleted_file: + if current_file is None or patch_info is None: + raise UnidiffParseError('Unexpected deleted file found: %s' % line) + current_file.target_file = DEV_NULL + patch_info.append(line) + continue + + # check for source file header + is_source_filename = RE_SOURCE_FILENAME.match(line) + if is_source_filename: + source_file = is_source_filename.group('filename') + source_timestamp = is_source_filename.group('timestamp') + # reset current file, unless we are processing a rename + # (in that case, source files should match) + if current_file is not None and not ( + current_file.source_file == source_file): + current_file = None + elif current_file is not None: + current_file.source_timestamp = source_timestamp + continue + + # check for target file header + is_target_filename = RE_TARGET_FILENAME.match(line) + if is_target_filename: + target_file = is_target_filename.group('filename') + target_timestamp = is_target_filename.group('timestamp') + if current_file is not None and not (current_file.target_file == target_file): + raise UnidiffParseError('Target without source: %s' % line) + if current_file is None: + # add current file to PatchSet + current_file = PatchedFile( + patch_info, source_file, target_file, + source_timestamp, target_timestamp) + self.append(current_file) + patch_info = None + else: + current_file.target_timestamp = target_timestamp + continue + + # check for hunk header + is_hunk_header = RE_HUNK_HEADER.match(line) + if is_hunk_header: + patch_info = None + if current_file is None: + raise UnidiffParseError('Unexpected hunk found: %s' % line) + current_file._parse_hunk(line, diff, encoding, metadata_only) + continue + + # check for no newline marker + is_no_newline = RE_NO_NEWLINE_MARKER.match(line) + if is_no_newline: + if current_file is None: + raise UnidiffParseError('Unexpected marker: %s' % line) + current_file._add_no_newline_marker_to_last_hunk() + continue + + # sometimes hunks can be followed by empty lines + if line == '\n' and current_file is not None: + current_file._append_trailing_empty_line() + continue + + # if nothing has matched above then this line is a patch info + if patch_info is None: + current_file = None + patch_info = PatchInfo() + + is_binary_diff = RE_BINARY_DIFF.match(line) + if is_binary_diff: + source_file = is_binary_diff.group('source_filename') + target_file = is_binary_diff.group('target_filename') + patch_info.append(line) + if current_file is not None: + current_file.is_binary_file = True + else: + current_file = PatchedFile( + patch_info, source_file, target_file, is_binary_file=True) + self.append(current_file) + patch_info = None + current_file = None + continue + + if line == 'GIT binary patch\n': + current_file.is_binary_file = True + patch_info = None + current_file = None + continue + + patch_info.append(line) + + @classmethod + def from_filename(cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None): + # type: (str, str, Optional[str]) -> PatchSet + """Return a PatchSet instance given a diff filename.""" + with open_file(filename, 'r', encoding=encoding, errors=errors, newline=newline) as f: + instance = cls(f) + return instance + + @staticmethod + def _convert_string(data, encoding=None, errors='strict'): + # type: (Union[str, bytes], str, str) -> StringIO + if encoding is not None: + # if encoding is given, assume bytes and decode + data = unicode(data, encoding=encoding, errors=errors) + return StringIO(data) + + @classmethod + def from_string(cls, data, encoding=None, errors='strict'): + # type: (str, str, Optional[str]) -> PatchSet + """Return a PatchSet instance given a diff string.""" + return cls(cls._convert_string(data, encoding, errors)) + + @property + def added_files(self): + # type: () -> list[PatchedFile] + """Return patch added files as a list.""" + return [f for f in self if f.is_added_file] + + @property + def removed_files(self): + # type: () -> list[PatchedFile] + """Return patch removed files as a list.""" + return [f for f in self if f.is_removed_file] + + @property + def modified_files(self): + # type: () -> list[PatchedFile] + """Return patch modified files as a list.""" + return [f for f in self if f.is_modified_file] + + @property + def added(self): + # type: () -> int + """Return the patch total added lines.""" + return sum([f.added for f in self]) + + @property + def removed(self): + # type: () -> int + """Return the patch total removed lines.""" + return sum([f.removed for f in self]) \ No newline at end of file diff --git a/scanpipe/pipes/unidiff/patch.py.ABOUT b/scanpipe/pipes/unidiff/patch.py.ABOUT new file mode 100644 index 0000000000..3118ed9643 --- /dev/null +++ b/scanpipe/pipes/unidiff/patch.py.ABOUT @@ -0,0 +1,13 @@ +about_resource: patch.py constants.py errors.py +name: patch +version: 0.7.5 +download_url: https://github.com/matiasb/python-unidiff/archive/refs/tags/v0.7.5.zip +description: Simple Python library to parse and interact with unified diff data. +homepage_url: https://github.com/matiasb/python-unidiff +license_expression: mit +attribute: yes +package_url: pkg:pypi/unidiff@0.7.5 +licenses: + - key: mit + name: MIT License + file: mit.LICENSE \ No newline at end of file diff --git a/scanpipe/pipes/unidiff/patch.py.LICENSE b/scanpipe/pipes/unidiff/patch.py.LICENSE new file mode 100644 index 0000000000..ca2c04c202 --- /dev/null +++ b/scanpipe/pipes/unidiff/patch.py.LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) +Copyright (c) 2012 Matias Bordese + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, +DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR +OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE +OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file From 09cfa5f95f4a8f2cdab58fc8e104c63f6c212b69 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 02:50:28 +0300 Subject: [PATCH 19/50] Update pipeline/functions name Get the dependency file and included in the PR Signed-off-by: ziad hany --- pyproject.toml | 2 +- ...ity.py => analyze_symbols_reachability.py} | 6 +- scanpipe/pipelines/find_vulnerabilities.py | 12 +- scanpipe/pipes/reachability.py | 12 +- scanpipe/pipes/symbols.py | 28 ++ scanpipe/pipes/unidiff/patch.py | 309 +++++++++++------- scanpipe/pipes/vulnerablecode.py | 7 +- .../tests/pipes/test_symbols_reachability.py | 4 +- 8 files changed, 237 insertions(+), 143 deletions(-) rename scanpipe/pipelines/{collect_symbols_reachability.py => analyze_symbols_reachability.py} (85%) diff --git a/pyproject.toml b/pyproject.toml index 50a4286150..d5dfbae829 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -152,7 +152,7 @@ run = "scancodeio:combined_run" analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker" analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS" analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows" -analyze_symbols_reachability = "scanpipe.pipelines.collect_symbols_reachability:SymbolReachability" +analyze_symbols_reachability = "scanpipe.pipelines.analyze_symbols_reachability:SymbolReachability" benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls" collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext" collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags" diff --git a/scanpipe/pipelines/collect_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py similarity index 85% rename from scanpipe/pipelines/collect_symbols_reachability.py rename to scanpipe/pipelines/analyze_symbols_reachability.py index c1d5fb11c4..2489df9c49 100644 --- a/scanpipe/pipelines/collect_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -20,14 +20,14 @@ class SymbolReachability(Pipeline): @classmethod def steps(cls): - return (cls.analyze_and_store_symbol_reachability,) + return (cls.analyze_symbol_reachability,) - def analyze_and_store_symbol_reachability(self): + def analyze_symbol_reachability(self): """ Perform symbol-level reachability analysis for each patch. This step compares the AST of patched/vulnerable files against the codebase resources. Results are stored directly in the 'extra_data' of each CodebaseResource. """ - reachability.collect_and_store_symbol_reachability_results( + reachability.analyze_and_store_symbol_reachability_results( project=self.project, logger=self.log ) diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index b0c8066b9a..1bdf8af6bc 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -19,7 +19,7 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. - +from aboutcode.pipeline import optional_step from scanpipe.pipelines import Pipeline from scanpipe.pipes import vulnerablecode @@ -34,11 +34,13 @@ class FindVulnerabilities(Pipeline): download_inputs = False is_addon = True results_url = "/project/{slug}/packages/?is_vulnerable=yes" + reachability = False @classmethod def steps(cls): return ( cls.check_vulnerablecode_service_availability, + cls.enable_reachability_analysis, cls.lookup_packages_vulnerabilities, cls.lookup_dependencies_vulnerabilities, ) @@ -48,6 +50,12 @@ def get_availability(cls): if not vulnerablecode.is_configured(): return "VulnerableCode is not configured." + @optional_step("reachability") + def enable_reachability_analysis(self): + """Enable the reachability flag for vulnerability lookups.""" + self.reachability = True + self.log("Reachability analysis is ENABLED.") + def check_vulnerablecode_service_availability(self): """Check if the VulnerableCode service if configured and available.""" if not vulnerablecode.is_configured(): @@ -62,6 +70,7 @@ def lookup_packages_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=packages, ignore_set=self.project.ignored_vulnerabilities_set, + reachability=self.reachability, logger=self.log, ) @@ -71,5 +80,6 @@ def lookup_dependencies_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=dependencies, ignore_set=self.project.ignored_vulnerabilities_set, + reachability=self.reachability, logger=self.log, ) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index d9e75f4cfa..46cda929ee 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -19,6 +19,8 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +# + import os import shutil import tempfile @@ -29,12 +31,12 @@ from git import Repo from git.diff import NULL_TREE from scancode.api import get_file_info -from unidiff import PatchSet from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint from scanpipe.pipes.symbols import is_supported_language +from scanpipe.pipes.unidiff.patch import PatchSet class ReachabilityStatus(str, Enum): @@ -100,8 +102,6 @@ def _cleanup(self): class PatchAnalyzer: - EMPTY_TREE_SHA = "4b825dc642cb6eb9a060e54bf8b8e6f9b79b4d2b" - def __init__(self, repo: Repo, commit_hash: str): self.repo = repo self.commit = repo.commit(commit_hash) @@ -145,7 +145,8 @@ def get_changed_files(self): return files def get_commit_diff_text(self): - base = self.parent_commit.hexsha if self.parent_commit else self.EMPTY_TREE_SHA + """Get the diff text, falling back to an empty tree if no parent exists.""" + base = self.parent_commit.hexsha if self.parent_commit else NULL_TREE return self.repo.git.diff(base, self.commit.hexsha, unified=3) @classmethod @@ -379,7 +380,6 @@ def generate_reachability_report(patch, repo, candidate_resources, logger=None): "reachability_status": classify_reachability(vuln_evidence).value, } } - print(report) existing = resource.extra_data.get("symbols_reachability") reports = existing if isinstance(existing, list) else [] @@ -398,7 +398,7 @@ def generate_reachability_report(patch, repo, candidate_resources, logger=None): ) -def collect_and_store_symbol_reachability_results(project, logger=None): +def analyze_and_store_symbol_reachability_results(project, logger=None): candidate_resources = project.codebaseresources.files().filter( is_binary=False, is_archive=False, is_media=False ) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 0f25819d51..f86630b4f5 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -353,8 +353,36 @@ class PythonTreeSitterQuery(LanguageQuery): syntax_config = {"self_keyword": "self", "separator": ".", "wildcard_symbol": "*"} +class JavaTreeSitterQuery(LanguageQuery): + language_name = "Java" + constants_query = ( + "(field_declaration declarator: " + "(variable_declarator name: (identifier) @name)) @constant" + ) + functions_query = """ + [(method_declaration name: (identifier) @name) + (constructor_declaration name: (identifier) @name)] @function + """ + classes_query = """ + [(class_declaration name: (identifier) @name) + (interface_declaration name: (identifier) @name) + (record_declaration name: (identifier) @name) + (enum_declaration name: (identifier) @name)] @class + """ + calls_query = """ + (method_invocation name: (identifier) @callee) + (method_invocation object: (_) @receiver name: (identifier) @callee) + """ + imports_query = """ + (import_declaration (scoped_identifier) @import_name) + (import_declaration (scoped_identifier) @module_name (asterisk) @import_name) + """ + syntax_config = {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"} + + TS_QUERIES = { "Python": PythonTreeSitterQuery, + "Java": JavaTreeSitterQuery, } diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py index c51bdbef79..8075a8cc77 100644 --- a/scanpipe/pipes/unidiff/patch.py +++ b/scanpipe/pipes/unidiff/patch.py @@ -1,4 +1,5 @@ -# Extracted essential patch code analyzer and modified from the original unidiff library: +# Extracted essential patch code analyzer and modified +# from the original unidiff library: # https://github.com/matiasb/python-unidiff/blob/2771a878f7bc6619e625feb4dbad3427f57f5237/unidiff/patch.py # -*- coding: utf-8 -*- @@ -24,13 +25,13 @@ # OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE # OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import unicode_literals import re from io import StringIO -from typing import Iterable, Optional, Union + class UnidiffParseError(Exception): ... + open_file = open make_str = str implements_to_string = lambda x: x @@ -38,65 +39,77 @@ class UnidiffParseError(Exception): ... basestring = str RE_SOURCE_FILENAME = re.compile( - r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') + r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' +) RE_TARGET_FILENAME = re.compile( - r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?') + r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' +) # check diff git line for git renamed files support RE_DIFF_GIT_HEADER = re.compile( - r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)') + r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)' +) RE_DIFF_GIT_HEADER_URI_LIKE = re.compile( - r'^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)') + r"^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)" +) RE_DIFF_GIT_HEADER_NO_PREFIX = re.compile( - r'^diff --git (?P[^\t\n]+) (?P[^\t\n]+)') + r"^diff --git (?P[^\t\n]+) (?P[^\t\n]+)" +) # check diff git new file marker `deleted file mode 100644` -RE_DIFF_GIT_DELETED_FILE = re.compile(r'^deleted file mode \d+$') +RE_DIFF_GIT_DELETED_FILE = re.compile(r"^deleted file mode \d+$") # check diff git new file marker `new file mode 100644` -RE_DIFF_GIT_NEW_FILE = re.compile(r'^new file mode \d+$') +RE_DIFF_GIT_NEW_FILE = re.compile(r"^new file mode \d+$") # @@ (source offset, length) (target offset, length) @@ (section header) -RE_HUNK_HEADER = re.compile( - r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") +RE_HUNK_HEADER = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") # kept line (context) # \n empty line (treat like context) # + added line # - deleted line # \ No newline case -RE_HUNK_BODY_LINE = re.compile( - r'^(?P[- \+\\])(?P.*)', re.DOTALL) +RE_HUNK_BODY_LINE = re.compile(r"^(?P[- \+\\])(?P.*)", re.DOTALL) RE_HUNK_EMPTY_BODY_LINE = re.compile( - r'^(?P[- \+\\]?)(?P[\r\n]{1,2})', re.DOTALL) + r"^(?P[- \+\\]?)(?P[\r\n]{1,2})", re.DOTALL +) -RE_NO_NEWLINE_MARKER = re.compile(r'^\\ No newline at end of file') +RE_NO_NEWLINE_MARKER = re.compile(r"^\\ No newline at end of file") RE_BINARY_DIFF = re.compile( - r'^Binary files? ' - r'(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?' - r'(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)') + r"^Binary files? " + r"(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?" + r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)" +) + +DEFAULT_ENCODING = "UTF-8" -DEFAULT_ENCODING = 'UTF-8' +DEV_NULL = "/dev/null" +LINE_TYPE_ADDED = "+" +LINE_TYPE_REMOVED = "-" +LINE_TYPE_CONTEXT = " " +LINE_TYPE_EMPTY = "" +LINE_TYPE_NO_NEWLINE = "\\" +LINE_VALUE_NO_NEWLINE = " No newline at end of file" -DEV_NULL = '/dev/null' -LINE_TYPE_ADDED = '+' -LINE_TYPE_REMOVED = '-' -LINE_TYPE_CONTEXT = ' ' -LINE_TYPE_EMPTY = '' -LINE_TYPE_NO_NEWLINE = '\\' -LINE_VALUE_NO_NEWLINE = ' No newline at end of file' @implements_to_string -class Line(object): +class Line: """A diff line.""" - def __init__(self, value, line_type, - source_line_no=None, target_line_no=None, diff_line_no=None): + def __init__( + self, + value, + line_type, + source_line_no=None, + target_line_no=None, + diff_line_no=None, + ): # type: (str, str, Optional[int], Optional[int], Optional[int]) -> None - super(Line, self).__init__() + super().__init__() self.source_line_no = source_line_no self.target_line_no = target_line_no self.diff_line_no = diff_line_no @@ -113,11 +126,13 @@ def __str__(self): def __eq__(self, other): # type: (Line) -> bool - return (self.source_line_no == other.source_line_no and - self.target_line_no == other.target_line_no and - self.diff_line_no == other.diff_line_no and - self.line_type == other.line_type and - self.value == other.value) + return ( + self.source_line_no == other.source_line_no + and self.target_line_no == other.target_line_no + and self.diff_line_no == other.diff_line_no + and self.line_type == other.line_type + and self.value == other.value + ) @property def is_added(self): @@ -137,7 +152,8 @@ def is_context(self): @implements_to_string class PatchInfo(list): - """Lines with extended patch info. + """ + Lines with extended patch info. Format of this info is not documented and it very much depends on patch producer. @@ -151,17 +167,18 @@ def __repr__(self): def __str__(self): # type: () -> str - return ''.join(unicode(line) for line in self) + return "".join(unicode(line) for line in self) @implements_to_string class Hunk(list): """Each of the modified blocks of a file.""" - def __init__(self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, - section_header=''): + def __init__( + self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, section_header="" + ): # type: (int, int, int, int, str) -> None - super(Hunk, self).__init__() + super().__init__() if src_len is None: src_len = 1 if tgt_len is None: @@ -176,21 +193,26 @@ def __init__(self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, def __repr__(self): # type: () -> str - value = "" % (self.source_start, - self.source_length, - self.target_start, - self.target_length, - self.section_header) + value = "" % ( + self.source_start, + self.source_length, + self.target_start, + self.target_length, + self.section_header, + ) return make_str(value) def __str__(self): # type: () -> str # section header is optional and thus we output it only if it's present head = "@@ -%d,%d +%d,%d @@%s\n" % ( - self.source_start, self.source_length, - self.target_start, self.target_length, - ' ' + self.section_header if self.section_header else '') - content = ''.join(unicode(line) for line in self) + self.source_start, + self.source_length, + self.target_start, + self.target_length, + " " + self.section_header if self.section_header else "", + ) + content = "".join(unicode(line) for line in self) return head + content def append(self, line): @@ -199,7 +221,7 @@ def append(self, line): # Make sure the line is encoded correctly. This is a no-op except for # potentially raising a UnicodeDecodeError. str(line) - super(Hunk, self).append(line) + super().append(line) @property def added(self): @@ -222,8 +244,10 @@ def removed(self): def is_valid(self): # type: () -> bool """Check hunk header data matches entered lines info.""" - return (len(self.source) == self.source_length and - len(self.target) == self.target_length) + return ( + len(self.source) == self.source_length + and len(self.target) == self.target_length + ) def source_lines(self): # type: () -> Iterable[Line] @@ -249,11 +273,17 @@ def target(self): class PatchedFile(list): """Patch updated file, it is a list of Hunks.""" - def __init__(self, patch_info=None, source='', target='', - source_timestamp=None, target_timestamp=None, - is_binary_file=False): + def __init__( + self, + patch_info=None, + source="", + target="", + source_timestamp=None, + target_timestamp=None, + is_binary_file=False, + ): # type: (Optional[PatchInfo], str, str, Optional[str], Optional[str], bool, bool) -> None - super(PatchedFile, self).__init__() + super().__init__() self.patch_info = patch_info self.source_file = source self.source_timestamp = source_timestamp @@ -267,18 +297,20 @@ def __repr__(self): def __str__(self): # type: () -> str - source = '' - target = '' + source = "" + target = "" # patch info is optional - info = '' if self.patch_info is None else str(self.patch_info) + info = "" if self.patch_info is None else str(self.patch_info) if not self.is_binary_file and self: source = "--- %s%s\n" % ( self.source_file, - '\t' + self.source_timestamp if self.source_timestamp else '') + "\t" + self.source_timestamp if self.source_timestamp else "", + ) target = "+++ %s%s\n" % ( self.target_file, - '\t' + self.target_timestamp if self.target_timestamp else '') - hunks = ''.join(unicode(hunk) for hunk in self) + "\t" + self.target_timestamp if self.target_timestamp else "", + ) + hunks = "".join(unicode(hunk) for hunk in self) return info + source + target + hunks def _parse_hunk(self, header, diff, encoding, metadata_only): @@ -302,12 +334,13 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): if metadata_only: # quick line type detection, no regex required line_type = line[0] if line else LINE_TYPE_CONTEXT - if line_type not in (LINE_TYPE_ADDED, - LINE_TYPE_REMOVED, - LINE_TYPE_CONTEXT, - LINE_TYPE_NO_NEWLINE): - raise UnidiffParseError( - 'Hunk diff line expected: %s' % line) + if line_type not in ( + LINE_TYPE_ADDED, + LINE_TYPE_REMOVED, + LINE_TYPE_CONTEXT, + LINE_TYPE_NO_NEWLINE, + ): + raise UnidiffParseError("Hunk diff line expected: %s" % line) if line_type == LINE_TYPE_ADDED: target_line_no += 1 @@ -329,14 +362,13 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) if not valid_line: - raise UnidiffParseError( - 'Hunk diff line expected: %s' % line) + raise UnidiffParseError("Hunk diff line expected: %s" % line) - line_type = valid_line.group('line_type') + line_type = valid_line.group("line_type") if line_type == LINE_TYPE_EMPTY: line_type = LINE_TYPE_CONTEXT - value = valid_line.group('value') # type: str + value = valid_line.group("value") # type: str original_line = Line(value, line_type=line_type) if line_type == LINE_TYPE_ADDED: @@ -356,23 +388,26 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): original_line = None # stop parsing if we got past expected number of lines - if (source_line_no > expected_source_end or - target_line_no > expected_target_end): - raise UnidiffParseError('Hunk is longer than expected') + if ( + source_line_no > expected_source_end + or target_line_no > expected_target_end + ): + raise UnidiffParseError("Hunk is longer than expected") if original_line: original_line.diff_line_no = diff_line_no hunk.append(original_line) # if hunk source/target lengths are ok, hunk is complete - if (source_line_no == expected_source_end and - target_line_no == expected_target_end): + if ( + source_line_no == expected_source_end + and target_line_no == expected_target_end + ): break # report an error if we haven't got expected number of lines - if (source_line_no < expected_source_end or - target_line_no < expected_target_end): - raise UnidiffParseError('Hunk is shorter than expected') + if source_line_no < expected_source_end or target_line_no < expected_target_end: + raise UnidiffParseError("Hunk is shorter than expected") if metadata_only: # HACK: set fixed calculated values when metadata_only is enabled @@ -384,18 +419,18 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): def _add_no_newline_marker_to_last_hunk(self): # type: () -> None if not self: - raise UnidiffParseError( - 'Unexpected marker:' + LINE_VALUE_NO_NEWLINE) + raise UnidiffParseError("Unexpected marker:" + LINE_VALUE_NO_NEWLINE) last_hunk = self[-1] last_hunk.append( - Line(LINE_VALUE_NO_NEWLINE + '\n', line_type=LINE_TYPE_NO_NEWLINE)) + Line(LINE_VALUE_NO_NEWLINE + "\n", line_type=LINE_TYPE_NO_NEWLINE) + ) def _append_trailing_empty_line(self): # type: () -> None if not self: - raise UnidiffParseError('Unexpected trailing newline character') + raise UnidiffParseError("Unexpected trailing newline character") last_hunk = self[-1] - last_hunk.append(Line('\n', line_type=LINE_TYPE_EMPTY)) + last_hunk.append(Line("\n", line_type=LINE_TYPE_EMPTY)) @property def path(self): @@ -403,7 +438,8 @@ def path(self): """Return the file path abstracted from VCS.""" filepath = self.source_file if filepath in (None, DEV_NULL) or ( - self.is_rename and self.target_file not in (None, DEV_NULL)): + self.is_rename and self.target_file not in (None, DEV_NULL) + ): # if this is a rename, prefer the target filename filepath = self.target_file @@ -411,11 +447,11 @@ def path(self): if quoted: filepath = filepath[1:-1] - if filepath.startswith('a/') or filepath.startswith('b/'): + if filepath.startswith("a/") or filepath.startswith("b/"): filepath = filepath[2:] if quoted: - filepath = '"{}"'.format(filepath) + filepath = f'"{filepath}"' return filepath @@ -433,9 +469,11 @@ def removed(self): @property def is_rename(self): - return (self.source_file != DEV_NULL + return ( + self.source_file != DEV_NULL and self.target_file != DEV_NULL - and self.source_file[2:] != self.target_file[2:]) + and self.source_file[2:] != self.target_file[2:] + ) @property def is_added_file(self): @@ -443,8 +481,9 @@ def is_added_file(self): """Return True if this patch adds the file.""" if self.source_file == DEV_NULL: return True - return (len(self) == 1 and self[0].source_start == 0 and - self[0].source_length == 0) + return ( + len(self) == 1 and self[0].source_start == 0 and self[0].source_length == 0 + ) @property def is_removed_file(self): @@ -452,8 +491,9 @@ def is_removed_file(self): """Return True if this patch removes the file.""" if self.target_file == DEV_NULL: return True - return (len(self) == 1 and self[0].target_start == 0 and - self[0].target_length == 0) + return ( + len(self) == 1 and self[0].target_start == 0 and self[0].target_length == 0 + ) @property def is_modified_file(self): @@ -468,7 +508,7 @@ class PatchSet(list): def __init__(self, f, encoding=None, metadata_only=False): # type: (Union[StringIO, str], Optional[str], bool) -> None - super(PatchSet, self).__init__() + super().__init__() # convert string inputs to StringIO objects if isinstance(f, basestring): @@ -484,11 +524,11 @@ def __init__(self, f, encoding=None, metadata_only=False): def __repr__(self): # type: () -> str - return make_str('') % super(PatchSet, self).__repr__() + return make_str("") % super().__repr__() def __str__(self): # type: () -> str - return ''.join(unicode(patched_file) for patched_file in self) + return "".join(unicode(patched_file) for patched_file in self) def _parse(self, diff, encoding, metadata_only): # type: (StringIO, Optional[str], bool) -> None @@ -501,15 +541,18 @@ def _parse(self, diff, encoding, metadata_only): line = line.decode(encoding) # check for a git file rename - is_diff_git_header = RE_DIFF_GIT_HEADER.match(line) or \ - RE_DIFF_GIT_HEADER_URI_LIKE.match(line) or \ - RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) + is_diff_git_header = ( + RE_DIFF_GIT_HEADER.match(line) + or RE_DIFF_GIT_HEADER_URI_LIKE.match(line) + or RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) + ) if is_diff_git_header: patch_info = PatchInfo() - source_file = is_diff_git_header.group('source') - target_file = is_diff_git_header.group('target') + source_file = is_diff_git_header.group("source") + target_file = is_diff_git_header.group("target") current_file = PatchedFile( - patch_info, source_file, target_file, None, None) + patch_info, source_file, target_file, None, None + ) self.append(current_file) patch_info.append(line) continue @@ -518,7 +561,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) if is_diff_git_new_file: if current_file is None or patch_info is None: - raise UnidiffParseError('Unexpected new file found: %s' % line) + raise UnidiffParseError("Unexpected new file found: %s" % line) current_file.source_file = DEV_NULL patch_info.append(line) continue @@ -527,7 +570,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) if is_diff_git_deleted_file: if current_file is None or patch_info is None: - raise UnidiffParseError('Unexpected deleted file found: %s' % line) + raise UnidiffParseError("Unexpected deleted file found: %s" % line) current_file.target_file = DEV_NULL patch_info.append(line) continue @@ -535,12 +578,13 @@ def _parse(self, diff, encoding, metadata_only): # check for source file header is_source_filename = RE_SOURCE_FILENAME.match(line) if is_source_filename: - source_file = is_source_filename.group('filename') - source_timestamp = is_source_filename.group('timestamp') + source_file = is_source_filename.group("filename") + source_timestamp = is_source_filename.group("timestamp") # reset current file, unless we are processing a rename # (in that case, source files should match) if current_file is not None and not ( - current_file.source_file == source_file): + current_file.source_file == source_file + ): current_file = None elif current_file is not None: current_file.source_timestamp = source_timestamp @@ -549,15 +593,21 @@ def _parse(self, diff, encoding, metadata_only): # check for target file header is_target_filename = RE_TARGET_FILENAME.match(line) if is_target_filename: - target_file = is_target_filename.group('filename') - target_timestamp = is_target_filename.group('timestamp') - if current_file is not None and not (current_file.target_file == target_file): - raise UnidiffParseError('Target without source: %s' % line) + target_file = is_target_filename.group("filename") + target_timestamp = is_target_filename.group("timestamp") + if current_file is not None and not ( + current_file.target_file == target_file + ): + raise UnidiffParseError("Target without source: %s" % line) if current_file is None: # add current file to PatchSet current_file = PatchedFile( - patch_info, source_file, target_file, - source_timestamp, target_timestamp) + patch_info, + source_file, + target_file, + source_timestamp, + target_timestamp, + ) self.append(current_file) patch_info = None else: @@ -569,7 +619,7 @@ def _parse(self, diff, encoding, metadata_only): if is_hunk_header: patch_info = None if current_file is None: - raise UnidiffParseError('Unexpected hunk found: %s' % line) + raise UnidiffParseError("Unexpected hunk found: %s" % line) current_file._parse_hunk(line, diff, encoding, metadata_only) continue @@ -577,12 +627,12 @@ def _parse(self, diff, encoding, metadata_only): is_no_newline = RE_NO_NEWLINE_MARKER.match(line) if is_no_newline: if current_file is None: - raise UnidiffParseError('Unexpected marker: %s' % line) + raise UnidiffParseError("Unexpected marker: %s" % line) current_file._add_no_newline_marker_to_last_hunk() continue # sometimes hunks can be followed by empty lines - if line == '\n' and current_file is not None: + if line == "\n" and current_file is not None: current_file._append_trailing_empty_line() continue @@ -593,20 +643,21 @@ def _parse(self, diff, encoding, metadata_only): is_binary_diff = RE_BINARY_DIFF.match(line) if is_binary_diff: - source_file = is_binary_diff.group('source_filename') - target_file = is_binary_diff.group('target_filename') + source_file = is_binary_diff.group("source_filename") + target_file = is_binary_diff.group("target_filename") patch_info.append(line) if current_file is not None: current_file.is_binary_file = True else: current_file = PatchedFile( - patch_info, source_file, target_file, is_binary_file=True) + patch_info, source_file, target_file, is_binary_file=True + ) self.append(current_file) patch_info = None current_file = None continue - if line == 'GIT binary patch\n': + if line == "GIT binary patch\n": current_file.is_binary_file = True patch_info = None current_file = None @@ -615,15 +666,19 @@ def _parse(self, diff, encoding, metadata_only): patch_info.append(line) @classmethod - def from_filename(cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None): + def from_filename( + cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None + ): # type: (str, str, Optional[str]) -> PatchSet """Return a PatchSet instance given a diff filename.""" - with open_file(filename, 'r', encoding=encoding, errors=errors, newline=newline) as f: + with open_file( + filename, "r", encoding=encoding, errors=errors, newline=newline + ) as f: instance = cls(f) return instance @staticmethod - def _convert_string(data, encoding=None, errors='strict'): + def _convert_string(data, encoding=None, errors="strict"): # type: (Union[str, bytes], str, str) -> StringIO if encoding is not None: # if encoding is given, assume bytes and decode @@ -631,7 +686,7 @@ def _convert_string(data, encoding=None, errors='strict'): return StringIO(data) @classmethod - def from_string(cls, data, encoding=None, errors='strict'): + def from_string(cls, data, encoding=None, errors="strict"): # type: (str, str, Optional[str]) -> PatchSet """Return a PatchSet instance given a diff string.""" return cls(cls._convert_string(data, encoding, errors)) @@ -664,4 +719,4 @@ def added(self): def removed(self): # type: () -> int """Return the patch total removed lines.""" - return sum([f.removed for f in self]) \ No newline at end of file + return sum([f.removed for f in self]) diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 4deaefc704..57924bff70 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -109,6 +109,7 @@ def request_post( def bulk_search_by_purl( purls, + reachability=False, timeout=None, api_url=VULNERABLECODE_API_URL, ): @@ -118,7 +119,7 @@ def bulk_search_by_purl( data = { "purls": purls, "details": True, - "reachability": True, + "reachability": reachability, } logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}") @@ -137,7 +138,7 @@ def filter_vulnerabilities(vulnerabilities, ignore_set): def fetch_vulnerabilities( - packages, chunk_size=1000, logger=logger.info, ignore_set=None + packages, chunk_size=1000, logger=logger.info, ignore_set=None, reachability=False ): """ Fetch and store vulnerabilities for each provided `packages`. @@ -147,7 +148,7 @@ def fetch_vulnerabilities( for purls_batch in chunked(get_purls(packages), chunk_size): try: - response_data = bulk_search_by_purl(purls_batch) + response_data = bulk_search_by_purl(purls_batch, reachability=reachability) except (requests.RequestException, ValueError, TypeError) as exception: logger(f"{label} [Exception] {exception}") return diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 620e886c05..ef054dd5a1 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -32,7 +32,7 @@ from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import collect_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import analyze_and_store_symbol_reachability_results from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor @@ -104,7 +104,7 @@ def test_get_symbol_reachability_results( resource.programming_language = lang resource.save() - collect_and_store_symbol_reachability_results(self.project1) + analyze_and_store_symbol_reachability_results(self.project1) resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") From 3bfa88aa371cff71242e1dfe66ef350237972bc8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 03:53:25 +0300 Subject: [PATCH 20/50] Fix CI files format Signed-off-by: ziad hany --- scanpipe/pipes/unidiff/patch.py | 75 +++++++++---------- .../tests/pipes/test_symbols_reachability.py | 2 +- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py index 8075a8cc77..e53c7c7d94 100644 --- a/scanpipe/pipes/unidiff/patch.py +++ b/scanpipe/pipes/unidiff/patch.py @@ -34,7 +34,12 @@ class UnidiffParseError(Exception): ... open_file = open make_str = str -implements_to_string = lambda x: x + + +def implements_to_string(x): + return x + + unicode = str basestring = str @@ -82,7 +87,8 @@ class UnidiffParseError(Exception): ... RE_BINARY_DIFF = re.compile( r"^Binary files? " r"(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?" - r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)? (differ|has changed)" + r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)?" + r" (differ|has changed)" ) DEFAULT_ENCODING = "UTF-8" @@ -122,7 +128,7 @@ def __repr__(self): def __str__(self): # type: () -> str - return "%s%s" % (self.line_type, self.value) + return f"{self.line_type}{self.value}" def __eq__(self, other): # type: (Line) -> bool @@ -162,7 +168,7 @@ class PatchInfo(list): def __repr__(self): # type: () -> str - value = "" % self[0].strip() + value = f"" return make_str(value) def __str__(self): @@ -193,24 +199,19 @@ def __init__( def __repr__(self): # type: () -> str - value = "" % ( - self.source_start, - self.source_length, - self.target_start, - self.target_length, - self.section_header, + value = ( + f"" ) return make_str(value) def __str__(self): # type: () -> str # section header is optional and thus we output it only if it's present - head = "@@ -%d,%d +%d,%d @@%s\n" % ( - self.source_start, - self.source_length, - self.target_start, - self.target_length, - " " + self.section_header if self.section_header else "", + section_hdr = f" {self.section_header}" if self.section_header else "" + head = ( + f"@@ -{self.source_start},{self.source_length} " + f"+{self.target_start},{self.target_length} @@{section_hdr}\n" ) content = "".join(unicode(line) for line in self) return head + content @@ -252,22 +253,22 @@ def is_valid(self): def source_lines(self): # type: () -> Iterable[Line] """Hunk lines from source file (generator).""" - return (l for l in self if l.is_context or l.is_removed) + return (line for line in self if line.is_context or line.is_removed) @property def source(self): # type: () -> Iterable[str] - return [str(l) for l in self.source_lines()] + return [str(line) for line in self.source_lines()] def target_lines(self): # type: () -> Iterable[Line] """Hunk lines from target file (generator).""" - return (l for l in self if l.is_context or l.is_added) + return (line for line in self if line.is_context or line.is_added) @property def target(self): # type: () -> Iterable[str] - return [str(l) for l in self.target_lines()] + return [str(line) for line in self.target_lines()] class PatchedFile(list): @@ -302,18 +303,16 @@ def __str__(self): # patch info is optional info = "" if self.patch_info is None else str(self.patch_info) if not self.is_binary_file and self: - source = "--- %s%s\n" % ( - self.source_file, - "\t" + self.source_timestamp if self.source_timestamp else "", - ) - target = "+++ %s%s\n" % ( - self.target_file, - "\t" + self.target_timestamp if self.target_timestamp else "", - ) + source_ts = f"\t{self.source_timestamp}" if self.source_timestamp else "" + source = f"--- {self.source_file}{source_ts}\n" + + target_ts = f"\t{self.target_timestamp}" if self.target_timestamp else "" + target = f"+++ {self.target_file}{target_ts}\n" + hunks = "".join(unicode(hunk) for hunk in self) return info + source + target + hunks - def _parse_hunk(self, header, diff, encoding, metadata_only): + def _parse_hunk(self, header, diff, encoding, metadata_only): # noqa: C901 # type: (str, enumerate[str], Optional[str], bool) -> None """Parse hunk details.""" header_info = RE_HUNK_HEADER.match(header) @@ -340,7 +339,7 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): LINE_TYPE_CONTEXT, LINE_TYPE_NO_NEWLINE, ): - raise UnidiffParseError("Hunk diff line expected: %s" % line) + raise UnidiffParseError(f"Hunk diff line expected: {line}") if line_type == LINE_TYPE_ADDED: target_line_no += 1 @@ -362,7 +361,7 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) if not valid_line: - raise UnidiffParseError("Hunk diff line expected: %s" % line) + raise UnidiffParseError(f"Hunk diff line expected: {line}") line_type = valid_line.group("line_type") if line_type == LINE_TYPE_EMPTY: @@ -410,7 +409,7 @@ def _parse_hunk(self, header, diff, encoding, metadata_only): raise UnidiffParseError("Hunk is shorter than expected") if metadata_only: - # HACK: set fixed calculated values when metadata_only is enabled + # set fixed calculated values when metadata_only is enabled hunk._added = added hunk._removed = removed @@ -530,7 +529,7 @@ def __str__(self): # type: () -> str return "".join(unicode(patched_file) for patched_file in self) - def _parse(self, diff, encoding, metadata_only): + def _parse(self, diff, encoding, metadata_only): # noqa: C901 # type: (StringIO, Optional[str], bool) -> None current_file = None patch_info = None @@ -561,7 +560,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) if is_diff_git_new_file: if current_file is None or patch_info is None: - raise UnidiffParseError("Unexpected new file found: %s" % line) + raise UnidiffParseError(f"Unexpected new file found: {line}") current_file.source_file = DEV_NULL patch_info.append(line) continue @@ -570,7 +569,7 @@ def _parse(self, diff, encoding, metadata_only): is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) if is_diff_git_deleted_file: if current_file is None or patch_info is None: - raise UnidiffParseError("Unexpected deleted file found: %s" % line) + raise UnidiffParseError(f"Unexpected deleted file found: {line}") current_file.target_file = DEV_NULL patch_info.append(line) continue @@ -598,7 +597,7 @@ def _parse(self, diff, encoding, metadata_only): if current_file is not None and not ( current_file.target_file == target_file ): - raise UnidiffParseError("Target without source: %s" % line) + raise UnidiffParseError(f"Target without source: {line}") if current_file is None: # add current file to PatchSet current_file = PatchedFile( @@ -619,7 +618,7 @@ def _parse(self, diff, encoding, metadata_only): if is_hunk_header: patch_info = None if current_file is None: - raise UnidiffParseError("Unexpected hunk found: %s" % line) + raise UnidiffParseError(f"Unexpected hunk found: {line}") current_file._parse_hunk(line, diff, encoding, metadata_only) continue @@ -627,7 +626,7 @@ def _parse(self, diff, encoding, metadata_only): is_no_newline = RE_NO_NEWLINE_MARKER.match(line) if is_no_newline: if current_file is None: - raise UnidiffParseError("Unexpected marker: %s" % line) + raise UnidiffParseError(f"Unexpected marker: {line}") current_file._add_no_newline_marker_to_last_hunk() continue diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index ef054dd5a1..a86081b1f3 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -31,8 +31,8 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus -from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.reachability import analyze_and_store_symbol_reachability_results +from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor From ec799431d653d2f1540c6239a72a0ddce0990366 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 04:02:18 +0300 Subject: [PATCH 21/50] Fix a typo in patch.py.ABOUT file Signed-off-by: ziad hany --- scanpipe/pipes/unidiff/patch.py.ABOUT | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/pipes/unidiff/patch.py.ABOUT b/scanpipe/pipes/unidiff/patch.py.ABOUT index 3118ed9643..75c3bb8ade 100644 --- a/scanpipe/pipes/unidiff/patch.py.ABOUT +++ b/scanpipe/pipes/unidiff/patch.py.ABOUT @@ -10,4 +10,4 @@ package_url: pkg:pypi/unidiff@0.7.5 licenses: - key: mit name: MIT License - file: mit.LICENSE \ No newline at end of file + file: patch.py.LICENSE \ No newline at end of file From 3744e2fc49afdab53c1d2ecade8cd6490d022ad1 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 05:05:15 +0300 Subject: [PATCH 22/50] Remove the unidiff library from the dependencies Signed-off-by: ziad hany --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index d5dfbae829..b309d0a955 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,7 +84,6 @@ dependencies = [ "urllib3==2.7.0", "idna==3.18", "GitPython==3.1.50", - "unidiff==0.7.5", "lxml==6.1.1", "certifi==2026.6.17", # Profiling From 128bfdfd7c013845fd4ab870071e3f9eb497b7a5 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 23 Jul 2026 05:27:26 +0300 Subject: [PATCH 23/50] Skip the test for macOS Signed-off-by: ziad hany --- scanpipe/tests/pipes/test_symbols_reachability.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index a86081b1f3..e24c42932b 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -19,8 +19,9 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. - +import sys from pathlib import Path +from unittest import skipIf from unittest.mock import MagicMock from unittest.mock import PropertyMock from unittest.mock import patch @@ -37,6 +38,7 @@ from scanpipe.pipes.symbols import SymbolExtractor +@skipIf(sys.platform == "darwin", "Not supported on macOS") class SymbolReachabilityPipesTest(TestCase): data = Path(__file__).parent.parent / "data" / "reachability" From 7258c0a680733890dcb25ab540e4d38cb60e26d7 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 29 Jul 2026 03:17:06 +0300 Subject: [PATCH 24/50] Allow reachability by default, for vulnerabilities pipeline Signed-off-by: ziad hany --- scanpipe/pipelines/find_vulnerabilities.py | 12 +----------- scanpipe/pipes/vulnerablecode.py | 7 +++---- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/scanpipe/pipelines/find_vulnerabilities.py b/scanpipe/pipelines/find_vulnerabilities.py index 1bdf8af6bc..b0c8066b9a 100644 --- a/scanpipe/pipelines/find_vulnerabilities.py +++ b/scanpipe/pipelines/find_vulnerabilities.py @@ -19,7 +19,7 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from aboutcode.pipeline import optional_step + from scanpipe.pipelines import Pipeline from scanpipe.pipes import vulnerablecode @@ -34,13 +34,11 @@ class FindVulnerabilities(Pipeline): download_inputs = False is_addon = True results_url = "/project/{slug}/packages/?is_vulnerable=yes" - reachability = False @classmethod def steps(cls): return ( cls.check_vulnerablecode_service_availability, - cls.enable_reachability_analysis, cls.lookup_packages_vulnerabilities, cls.lookup_dependencies_vulnerabilities, ) @@ -50,12 +48,6 @@ def get_availability(cls): if not vulnerablecode.is_configured(): return "VulnerableCode is not configured." - @optional_step("reachability") - def enable_reachability_analysis(self): - """Enable the reachability flag for vulnerability lookups.""" - self.reachability = True - self.log("Reachability analysis is ENABLED.") - def check_vulnerablecode_service_availability(self): """Check if the VulnerableCode service if configured and available.""" if not vulnerablecode.is_configured(): @@ -70,7 +62,6 @@ def lookup_packages_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=packages, ignore_set=self.project.ignored_vulnerabilities_set, - reachability=self.reachability, logger=self.log, ) @@ -80,6 +71,5 @@ def lookup_dependencies_vulnerabilities(self): vulnerablecode.fetch_vulnerabilities( packages=dependencies, ignore_set=self.project.ignored_vulnerabilities_set, - reachability=self.reachability, logger=self.log, ) diff --git a/scanpipe/pipes/vulnerablecode.py b/scanpipe/pipes/vulnerablecode.py index 57924bff70..4deaefc704 100644 --- a/scanpipe/pipes/vulnerablecode.py +++ b/scanpipe/pipes/vulnerablecode.py @@ -109,7 +109,6 @@ def request_post( def bulk_search_by_purl( purls, - reachability=False, timeout=None, api_url=VULNERABLECODE_API_URL, ): @@ -119,7 +118,7 @@ def bulk_search_by_purl( data = { "purls": purls, "details": True, - "reachability": reachability, + "reachability": True, } logger.debug(f"VulnerableCode: url={url} purls_count={len(purls)}") @@ -138,7 +137,7 @@ def filter_vulnerabilities(vulnerabilities, ignore_set): def fetch_vulnerabilities( - packages, chunk_size=1000, logger=logger.info, ignore_set=None, reachability=False + packages, chunk_size=1000, logger=logger.info, ignore_set=None ): """ Fetch and store vulnerabilities for each provided `packages`. @@ -148,7 +147,7 @@ def fetch_vulnerabilities( for purls_batch in chunked(get_purls(packages), chunk_size): try: - response_data = bulk_search_by_purl(purls_batch, reachability=reachability) + response_data = bulk_search_by_purl(purls_batch) except (requests.RequestException, ValueError, TypeError) as exception: logger(f"{label} [Exception] {exception}") return From 27de9f1c915d2972fbcd271402620ab7b623f29b Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 30 Jul 2026 02:23:55 +0300 Subject: [PATCH 25/50] Update the code to difflib instead of unidiff library Fix a bug related to constant detections and add a test Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 73 +- scanpipe/pipes/unidiff/patch.py | 721 ------------------ scanpipe/pipes/unidiff/patch.py.ABOUT | 13 - scanpipe/pipes/unidiff/patch.py.LICENSE | 20 - scanpipe/tests/data/reachability/app.py | 2 + .../tests/data/reachability/diff-app.patch | 39 - scanpipe/tests/data/reachability/fixed-app.py | 2 + scanpipe/tests/data/reachability/vuln-app.py | 2 + .../tests/pipes/test_symbols_reachability.py | 194 +++-- 9 files changed, 146 insertions(+), 920 deletions(-) delete mode 100644 scanpipe/pipes/unidiff/patch.py delete mode 100644 scanpipe/pipes/unidiff/patch.py.ABOUT delete mode 100644 scanpipe/pipes/unidiff/patch.py.LICENSE delete mode 100644 scanpipe/tests/data/reachability/diff-app.patch diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 46cda929ee..6c7768dbad 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -20,7 +20,7 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. # - +import difflib import os import shutil import tempfile @@ -36,7 +36,6 @@ from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint from scanpipe.pipes.symbols import is_supported_language -from scanpipe.pipes.unidiff.patch import PatchSet class ReachabilityStatus(str, Enum): @@ -149,6 +148,26 @@ def get_commit_diff_text(self): base = self.parent_commit.hexsha if self.parent_commit else NULL_TREE return self.repo.git.diff(base, self.commit.hexsha, unified=3) + @classmethod + def compute_changed_lines(cls, vulnerable_text, fixed_text): + """Return the removed and added line numbers between two file contents.""" + matcher = difflib.SequenceMatcher( + a=vulnerable_text.splitlines(), + b=fixed_text.splitlines(), + autojunk=False, # otherwise difflib ignores lines that repeat often + ) + + removed_lines = [] + added_lines = [] + for tag, vuln_start, vuln_end, fixed_start, fixed_end in matcher.get_opcodes(): + if tag == "equal": + continue + # opcodes are 0-based and end-exclusive, line numbers are 1-based + removed_lines.extend(range(vuln_start + 1, vuln_end + 1)) + added_lines.extend(range(fixed_start + 1, fixed_end + 1)) + + return removed_lines, added_lines + @classmethod def diff_changed_symbols(cls, vuln_meta, fixed_meta): vuln_only = { @@ -163,50 +182,16 @@ def diff_changed_symbols(cls, vuln_meta, fixed_meta): } return vuln_only, fixed_only - @classmethod - def parse_diff_lines(cls, diff_text): - diff_map = {} - if not diff_text: - return diff_map - - for patched_file in PatchSet.from_string(diff_text): - removed = [] - added = [] - - for hunk in patched_file: - removed.extend( - line.source_line_no - for line in hunk - if line.is_removed and line.source_line_no - ) - added.extend( - line.target_line_no - for line in hunk - if line.is_added and line.target_line_no - ) - - candidates = { - patched_file.path, - (patched_file.source_file or "").removeprefix("a/"), - (patched_file.target_file or "").removeprefix("b/"), - } - - for candidate in candidates: - if candidate: - diff_map[candidate] = (removed, added) - - return diff_map - def collect_patch_symbols(self): - diff_text = self.get_commit_diff_text() + by_language = {} changed_files = self.get_changed_files() - diff_line_map = self.parse_diff_lines(diff_text=diff_text) - by_language = {} for file_path, texts in changed_files.items(): vulnerable_text = texts["vulnerable_text"] fixed_text = texts["fixed_text"] - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + removed_lines, added_lines = self.compute_changed_lines( + vulnerable_text, fixed_text + ) vuln_meta, fixed_meta, language = self.analyze( vulnerable_text=vulnerable_text, @@ -504,11 +489,9 @@ def build_index(self) -> dict | None: ) for node, _ in lang_query.get_constants(tree.root_node): - # Skip constants defined inside functions, classes, or other blocks - if node.parent == tree.root_node: - self.process_node( - node, extractor, definitions_index, definitions, fingerprints - ) + self.process_node( + node, extractor, definitions_index, definitions, fingerprints + ) return { "definitions": definitions, diff --git a/scanpipe/pipes/unidiff/patch.py b/scanpipe/pipes/unidiff/patch.py deleted file mode 100644 index e53c7c7d94..0000000000 --- a/scanpipe/pipes/unidiff/patch.py +++ /dev/null @@ -1,721 +0,0 @@ -# Extracted essential patch code analyzer and modified -# from the original unidiff library: -# https://github.com/matiasb/python-unidiff/blob/2771a878f7bc6619e625feb4dbad3427f57f5237/unidiff/patch.py - -# -*- coding: utf-8 -*- - -# The MIT License (MIT) -# Copyright (c) 2014-2023 Matias Bordese -# -# Permission is hereby granted, free of charge, to any person obtaining a copy -# of this software and associated documentation files (the "Software"), to deal -# in the Software without restriction, including without limitation the rights -# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -# copies of the Software, and to permit persons to whom the Software is -# furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in -# all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -# OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -# OR OTHER DEALINGS IN THE SOFTWARE. - -import re -from io import StringIO - - -class UnidiffParseError(Exception): ... - - -open_file = open -make_str = str - - -def implements_to_string(x): - return x - - -unicode = str -basestring = str - -RE_SOURCE_FILENAME = re.compile( - r'^--- (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' -) -RE_TARGET_FILENAME = re.compile( - r'^\+\+\+ (?P"?[^\t\n]+"?)(?:\t(?P[^\n]+))?' -) - - -# check diff git line for git renamed files support -RE_DIFF_GIT_HEADER = re.compile( - r'^diff --git (?P"?a/[^\t\n]+"?) (?P"?b/[^\t\n]+"?)' -) -RE_DIFF_GIT_HEADER_URI_LIKE = re.compile( - r"^diff --git (?P.*://[^\t\n]+) (?P.*://[^\t\n]+)" -) -RE_DIFF_GIT_HEADER_NO_PREFIX = re.compile( - r"^diff --git (?P[^\t\n]+) (?P[^\t\n]+)" -) - -# check diff git new file marker `deleted file mode 100644` -RE_DIFF_GIT_DELETED_FILE = re.compile(r"^deleted file mode \d+$") - -# check diff git new file marker `new file mode 100644` -RE_DIFF_GIT_NEW_FILE = re.compile(r"^new file mode \d+$") - - -# @@ (source offset, length) (target offset, length) @@ (section header) -RE_HUNK_HEADER = re.compile(r"^@@ -(\d+)(?:,(\d+))? \+(\d+)(?:,(\d+))?\ @@[ ]?(.*)") - -# kept line (context) -# \n empty line (treat like context) -# + added line -# - deleted line -# \ No newline case -RE_HUNK_BODY_LINE = re.compile(r"^(?P[- \+\\])(?P.*)", re.DOTALL) -RE_HUNK_EMPTY_BODY_LINE = re.compile( - r"^(?P[- \+\\]?)(?P[\r\n]{1,2})", re.DOTALL -) - -RE_NO_NEWLINE_MARKER = re.compile(r"^\\ No newline at end of file") - -RE_BINARY_DIFF = re.compile( - r"^Binary files? " - r"(?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?" - r"(?: and (?P[^\t]+?)(?:\t(?P[\s0-9:\+-]+))?)?" - r" (differ|has changed)" -) - -DEFAULT_ENCODING = "UTF-8" - -DEV_NULL = "/dev/null" -LINE_TYPE_ADDED = "+" -LINE_TYPE_REMOVED = "-" -LINE_TYPE_CONTEXT = " " -LINE_TYPE_EMPTY = "" -LINE_TYPE_NO_NEWLINE = "\\" -LINE_VALUE_NO_NEWLINE = " No newline at end of file" - - -@implements_to_string -class Line: - """A diff line.""" - - def __init__( - self, - value, - line_type, - source_line_no=None, - target_line_no=None, - diff_line_no=None, - ): - # type: (str, str, Optional[int], Optional[int], Optional[int]) -> None - super().__init__() - self.source_line_no = source_line_no - self.target_line_no = target_line_no - self.diff_line_no = diff_line_no - self.line_type = line_type - self.value = value - - def __repr__(self): - # type: () -> str - return make_str("") % (self.line_type, self.value) - - def __str__(self): - # type: () -> str - return f"{self.line_type}{self.value}" - - def __eq__(self, other): - # type: (Line) -> bool - return ( - self.source_line_no == other.source_line_no - and self.target_line_no == other.target_line_no - and self.diff_line_no == other.diff_line_no - and self.line_type == other.line_type - and self.value == other.value - ) - - @property - def is_added(self): - # type: () -> bool - return self.line_type == LINE_TYPE_ADDED - - @property - def is_removed(self): - # type: () -> bool - return self.line_type == LINE_TYPE_REMOVED - - @property - def is_context(self): - # type: () -> bool - return self.line_type == LINE_TYPE_CONTEXT - - -@implements_to_string -class PatchInfo(list): - """ - Lines with extended patch info. - - Format of this info is not documented and it very much depends on - patch producer. - - """ - - def __repr__(self): - # type: () -> str - value = f"" - return make_str(value) - - def __str__(self): - # type: () -> str - return "".join(unicode(line) for line in self) - - -@implements_to_string -class Hunk(list): - """Each of the modified blocks of a file.""" - - def __init__( - self, src_start=0, src_len=0, tgt_start=0, tgt_len=0, section_header="" - ): - # type: (int, int, int, int, str) -> None - super().__init__() - if src_len is None: - src_len = 1 - if tgt_len is None: - tgt_len = 1 - self.source_start = int(src_start) - self.source_length = int(src_len) - self.target_start = int(tgt_start) - self.target_length = int(tgt_len) - self.section_header = section_header - self._added = None # Optional[int] - self._removed = None # Optional[int] - - def __repr__(self): - # type: () -> str - value = ( - f"" - ) - return make_str(value) - - def __str__(self): - # type: () -> str - # section header is optional and thus we output it only if it's present - section_hdr = f" {self.section_header}" if self.section_header else "" - head = ( - f"@@ -{self.source_start},{self.source_length} " - f"+{self.target_start},{self.target_length} @@{section_hdr}\n" - ) - content = "".join(unicode(line) for line in self) - return head + content - - def append(self, line): - # type: (Line) -> None - """Append the line to hunk, and keep track of source/target lines.""" - # Make sure the line is encoded correctly. This is a no-op except for - # potentially raising a UnicodeDecodeError. - str(line) - super().append(line) - - @property - def added(self): - # type: () -> Optional[int] - if self._added is not None: - return self._added - # re-calculate each time to allow for hunk modifications - # (which should mean metadata_only switch wasn't used) - return sum(1 for line in self if line.is_added) - - @property - def removed(self): - # type: () -> Optional[int] - if self._removed is not None: - return self._removed - # re-calculate each time to allow for hunk modifications - # (which should mean metadata_only switch wasn't used) - return sum(1 for line in self if line.is_removed) - - def is_valid(self): - # type: () -> bool - """Check hunk header data matches entered lines info.""" - return ( - len(self.source) == self.source_length - and len(self.target) == self.target_length - ) - - def source_lines(self): - # type: () -> Iterable[Line] - """Hunk lines from source file (generator).""" - return (line for line in self if line.is_context or line.is_removed) - - @property - def source(self): - # type: () -> Iterable[str] - return [str(line) for line in self.source_lines()] - - def target_lines(self): - # type: () -> Iterable[Line] - """Hunk lines from target file (generator).""" - return (line for line in self if line.is_context or line.is_added) - - @property - def target(self): - # type: () -> Iterable[str] - return [str(line) for line in self.target_lines()] - - -class PatchedFile(list): - """Patch updated file, it is a list of Hunks.""" - - def __init__( - self, - patch_info=None, - source="", - target="", - source_timestamp=None, - target_timestamp=None, - is_binary_file=False, - ): - # type: (Optional[PatchInfo], str, str, Optional[str], Optional[str], bool, bool) -> None - super().__init__() - self.patch_info = patch_info - self.source_file = source - self.source_timestamp = source_timestamp - self.target_file = target - self.target_timestamp = target_timestamp - self.is_binary_file = is_binary_file - - def __repr__(self): - # type: () -> str - return make_str("") % make_str(self.path) - - def __str__(self): - # type: () -> str - source = "" - target = "" - # patch info is optional - info = "" if self.patch_info is None else str(self.patch_info) - if not self.is_binary_file and self: - source_ts = f"\t{self.source_timestamp}" if self.source_timestamp else "" - source = f"--- {self.source_file}{source_ts}\n" - - target_ts = f"\t{self.target_timestamp}" if self.target_timestamp else "" - target = f"+++ {self.target_file}{target_ts}\n" - - hunks = "".join(unicode(hunk) for hunk in self) - return info + source + target + hunks - - def _parse_hunk(self, header, diff, encoding, metadata_only): # noqa: C901 - # type: (str, enumerate[str], Optional[str], bool) -> None - """Parse hunk details.""" - header_info = RE_HUNK_HEADER.match(header) - hunk_info = header_info.groups() - hunk = Hunk(*hunk_info) - - source_line_no = hunk.source_start - target_line_no = hunk.target_start - expected_source_end = source_line_no + hunk.source_length - expected_target_end = target_line_no + hunk.target_length - added = 0 - removed = 0 - - for diff_line_no, line in diff: - if encoding is not None: - line = line.decode(encoding) - - if metadata_only: - # quick line type detection, no regex required - line_type = line[0] if line else LINE_TYPE_CONTEXT - if line_type not in ( - LINE_TYPE_ADDED, - LINE_TYPE_REMOVED, - LINE_TYPE_CONTEXT, - LINE_TYPE_NO_NEWLINE, - ): - raise UnidiffParseError(f"Hunk diff line expected: {line}") - - if line_type == LINE_TYPE_ADDED: - target_line_no += 1 - added += 1 - elif line_type == LINE_TYPE_REMOVED: - source_line_no += 1 - removed += 1 - elif line_type == LINE_TYPE_CONTEXT: - target_line_no += 1 - source_line_no += 1 - - # no file content tracking - original_line = None - - else: - # parse diff line content - valid_line = RE_HUNK_BODY_LINE.match(line) - if not valid_line: - valid_line = RE_HUNK_EMPTY_BODY_LINE.match(line) - - if not valid_line: - raise UnidiffParseError(f"Hunk diff line expected: {line}") - - line_type = valid_line.group("line_type") - if line_type == LINE_TYPE_EMPTY: - line_type = LINE_TYPE_CONTEXT - - value = valid_line.group("value") # type: str - original_line = Line(value, line_type=line_type) - - if line_type == LINE_TYPE_ADDED: - original_line.target_line_no = target_line_no - target_line_no += 1 - elif line_type == LINE_TYPE_REMOVED: - original_line.source_line_no = source_line_no - source_line_no += 1 - elif line_type == LINE_TYPE_CONTEXT: - original_line.target_line_no = target_line_no - original_line.source_line_no = source_line_no - target_line_no += 1 - source_line_no += 1 - elif line_type == LINE_TYPE_NO_NEWLINE: - pass - else: - original_line = None - - # stop parsing if we got past expected number of lines - if ( - source_line_no > expected_source_end - or target_line_no > expected_target_end - ): - raise UnidiffParseError("Hunk is longer than expected") - - if original_line: - original_line.diff_line_no = diff_line_no - hunk.append(original_line) - - # if hunk source/target lengths are ok, hunk is complete - if ( - source_line_no == expected_source_end - and target_line_no == expected_target_end - ): - break - - # report an error if we haven't got expected number of lines - if source_line_no < expected_source_end or target_line_no < expected_target_end: - raise UnidiffParseError("Hunk is shorter than expected") - - if metadata_only: - # set fixed calculated values when metadata_only is enabled - hunk._added = added - hunk._removed = removed - - self.append(hunk) - - def _add_no_newline_marker_to_last_hunk(self): - # type: () -> None - if not self: - raise UnidiffParseError("Unexpected marker:" + LINE_VALUE_NO_NEWLINE) - last_hunk = self[-1] - last_hunk.append( - Line(LINE_VALUE_NO_NEWLINE + "\n", line_type=LINE_TYPE_NO_NEWLINE) - ) - - def _append_trailing_empty_line(self): - # type: () -> None - if not self: - raise UnidiffParseError("Unexpected trailing newline character") - last_hunk = self[-1] - last_hunk.append(Line("\n", line_type=LINE_TYPE_EMPTY)) - - @property - def path(self): - # type: () -> str - """Return the file path abstracted from VCS.""" - filepath = self.source_file - if filepath in (None, DEV_NULL) or ( - self.is_rename and self.target_file not in (None, DEV_NULL) - ): - # if this is a rename, prefer the target filename - filepath = self.target_file - - quoted = filepath.startswith('"') and filepath.endswith('"') - if quoted: - filepath = filepath[1:-1] - - if filepath.startswith("a/") or filepath.startswith("b/"): - filepath = filepath[2:] - - if quoted: - filepath = f'"{filepath}"' - - return filepath - - @property - def added(self): - # type: () -> int - """Return the file total added lines.""" - return sum([hunk.added for hunk in self]) - - @property - def removed(self): - # type: () -> int - """Return the file total removed lines.""" - return sum([hunk.removed for hunk in self]) - - @property - def is_rename(self): - return ( - self.source_file != DEV_NULL - and self.target_file != DEV_NULL - and self.source_file[2:] != self.target_file[2:] - ) - - @property - def is_added_file(self): - # type: () -> bool - """Return True if this patch adds the file.""" - if self.source_file == DEV_NULL: - return True - return ( - len(self) == 1 and self[0].source_start == 0 and self[0].source_length == 0 - ) - - @property - def is_removed_file(self): - # type: () -> bool - """Return True if this patch removes the file.""" - if self.target_file == DEV_NULL: - return True - return ( - len(self) == 1 and self[0].target_start == 0 and self[0].target_length == 0 - ) - - @property - def is_modified_file(self): - # type: () -> bool - """Return True if this patch modifies the file.""" - return not (self.is_added_file or self.is_removed_file) - - -@implements_to_string -class PatchSet(list): - """A list of PatchedFiles.""" - - def __init__(self, f, encoding=None, metadata_only=False): - # type: (Union[StringIO, str], Optional[str], bool) -> None - super().__init__() - - # convert string inputs to StringIO objects - if isinstance(f, basestring): - f = self._convert_string(f, encoding) # type: StringIO - - # make sure we pass an iterator object to parse - data = iter(f) - # if encoding is None, assume we are reading unicode data - # when metadata_only is True, only perform a minimal metadata parsing - # (ie. hunks without content) which is around 2.5-6 times faster; - # it will still validate the diff metadata consistency and get counts - self._parse(data, encoding=encoding, metadata_only=metadata_only) - - def __repr__(self): - # type: () -> str - return make_str("") % super().__repr__() - - def __str__(self): - # type: () -> str - return "".join(unicode(patched_file) for patched_file in self) - - def _parse(self, diff, encoding, metadata_only): # noqa: C901 - # type: (StringIO, Optional[str], bool) -> None - current_file = None - patch_info = None - - diff = enumerate(diff, 1) - for unused_diff_line_no, line in diff: - if encoding is not None: - line = line.decode(encoding) - - # check for a git file rename - is_diff_git_header = ( - RE_DIFF_GIT_HEADER.match(line) - or RE_DIFF_GIT_HEADER_URI_LIKE.match(line) - or RE_DIFF_GIT_HEADER_NO_PREFIX.match(line) - ) - if is_diff_git_header: - patch_info = PatchInfo() - source_file = is_diff_git_header.group("source") - target_file = is_diff_git_header.group("target") - current_file = PatchedFile( - patch_info, source_file, target_file, None, None - ) - self.append(current_file) - patch_info.append(line) - continue - - # check for a git new file - is_diff_git_new_file = RE_DIFF_GIT_NEW_FILE.match(line) - if is_diff_git_new_file: - if current_file is None or patch_info is None: - raise UnidiffParseError(f"Unexpected new file found: {line}") - current_file.source_file = DEV_NULL - patch_info.append(line) - continue - - # check for a git deleted file - is_diff_git_deleted_file = RE_DIFF_GIT_DELETED_FILE.match(line) - if is_diff_git_deleted_file: - if current_file is None or patch_info is None: - raise UnidiffParseError(f"Unexpected deleted file found: {line}") - current_file.target_file = DEV_NULL - patch_info.append(line) - continue - - # check for source file header - is_source_filename = RE_SOURCE_FILENAME.match(line) - if is_source_filename: - source_file = is_source_filename.group("filename") - source_timestamp = is_source_filename.group("timestamp") - # reset current file, unless we are processing a rename - # (in that case, source files should match) - if current_file is not None and not ( - current_file.source_file == source_file - ): - current_file = None - elif current_file is not None: - current_file.source_timestamp = source_timestamp - continue - - # check for target file header - is_target_filename = RE_TARGET_FILENAME.match(line) - if is_target_filename: - target_file = is_target_filename.group("filename") - target_timestamp = is_target_filename.group("timestamp") - if current_file is not None and not ( - current_file.target_file == target_file - ): - raise UnidiffParseError(f"Target without source: {line}") - if current_file is None: - # add current file to PatchSet - current_file = PatchedFile( - patch_info, - source_file, - target_file, - source_timestamp, - target_timestamp, - ) - self.append(current_file) - patch_info = None - else: - current_file.target_timestamp = target_timestamp - continue - - # check for hunk header - is_hunk_header = RE_HUNK_HEADER.match(line) - if is_hunk_header: - patch_info = None - if current_file is None: - raise UnidiffParseError(f"Unexpected hunk found: {line}") - current_file._parse_hunk(line, diff, encoding, metadata_only) - continue - - # check for no newline marker - is_no_newline = RE_NO_NEWLINE_MARKER.match(line) - if is_no_newline: - if current_file is None: - raise UnidiffParseError(f"Unexpected marker: {line}") - current_file._add_no_newline_marker_to_last_hunk() - continue - - # sometimes hunks can be followed by empty lines - if line == "\n" and current_file is not None: - current_file._append_trailing_empty_line() - continue - - # if nothing has matched above then this line is a patch info - if patch_info is None: - current_file = None - patch_info = PatchInfo() - - is_binary_diff = RE_BINARY_DIFF.match(line) - if is_binary_diff: - source_file = is_binary_diff.group("source_filename") - target_file = is_binary_diff.group("target_filename") - patch_info.append(line) - if current_file is not None: - current_file.is_binary_file = True - else: - current_file = PatchedFile( - patch_info, source_file, target_file, is_binary_file=True - ) - self.append(current_file) - patch_info = None - current_file = None - continue - - if line == "GIT binary patch\n": - current_file.is_binary_file = True - patch_info = None - current_file = None - continue - - patch_info.append(line) - - @classmethod - def from_filename( - cls, filename, encoding=DEFAULT_ENCODING, errors=None, newline=None - ): - # type: (str, str, Optional[str]) -> PatchSet - """Return a PatchSet instance given a diff filename.""" - with open_file( - filename, "r", encoding=encoding, errors=errors, newline=newline - ) as f: - instance = cls(f) - return instance - - @staticmethod - def _convert_string(data, encoding=None, errors="strict"): - # type: (Union[str, bytes], str, str) -> StringIO - if encoding is not None: - # if encoding is given, assume bytes and decode - data = unicode(data, encoding=encoding, errors=errors) - return StringIO(data) - - @classmethod - def from_string(cls, data, encoding=None, errors="strict"): - # type: (str, str, Optional[str]) -> PatchSet - """Return a PatchSet instance given a diff string.""" - return cls(cls._convert_string(data, encoding, errors)) - - @property - def added_files(self): - # type: () -> list[PatchedFile] - """Return patch added files as a list.""" - return [f for f in self if f.is_added_file] - - @property - def removed_files(self): - # type: () -> list[PatchedFile] - """Return patch removed files as a list.""" - return [f for f in self if f.is_removed_file] - - @property - def modified_files(self): - # type: () -> list[PatchedFile] - """Return patch modified files as a list.""" - return [f for f in self if f.is_modified_file] - - @property - def added(self): - # type: () -> int - """Return the patch total added lines.""" - return sum([f.added for f in self]) - - @property - def removed(self): - # type: () -> int - """Return the patch total removed lines.""" - return sum([f.removed for f in self]) diff --git a/scanpipe/pipes/unidiff/patch.py.ABOUT b/scanpipe/pipes/unidiff/patch.py.ABOUT deleted file mode 100644 index 75c3bb8ade..0000000000 --- a/scanpipe/pipes/unidiff/patch.py.ABOUT +++ /dev/null @@ -1,13 +0,0 @@ -about_resource: patch.py constants.py errors.py -name: patch -version: 0.7.5 -download_url: https://github.com/matiasb/python-unidiff/archive/refs/tags/v0.7.5.zip -description: Simple Python library to parse and interact with unified diff data. -homepage_url: https://github.com/matiasb/python-unidiff -license_expression: mit -attribute: yes -package_url: pkg:pypi/unidiff@0.7.5 -licenses: - - key: mit - name: MIT License - file: patch.py.LICENSE \ No newline at end of file diff --git a/scanpipe/pipes/unidiff/patch.py.LICENSE b/scanpipe/pipes/unidiff/patch.py.LICENSE deleted file mode 100644 index ca2c04c202..0000000000 --- a/scanpipe/pipes/unidiff/patch.py.LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) -Copyright (c) 2012 Matias Bordese - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, -DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE -OR OTHER DEALINGS IN THE SOFTWARE. \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/app.py index b8c9eff5e0..32eebc2fa1 100644 --- a/scanpipe/tests/data/reachability/app.py +++ b/scanpipe/tests/data/reachability/app.py @@ -1,5 +1,7 @@ import os +debug = False + class ReportGenerator: """A dummy class to test AST class method parsing.""" diff --git a/scanpipe/tests/data/reachability/diff-app.patch b/scanpipe/tests/data/reachability/diff-app.patch deleted file mode 100644 index ccb86953a8..0000000000 --- a/scanpipe/tests/data/reachability/diff-app.patch +++ /dev/null @@ -1,39 +0,0 @@ -From 8f7b1c3d9a4e2b6f5d8c1a2e3f4b5c6d7e8f9a0b Mon Sep 17 00:00:00 2001 -From: Security Team -Date: Tue, 2 Jun 2026 10:00:00 +0000 -Subject: [PATCH] Fix path traversal vulnerability in report generator - -- Validates that target paths stay within the designated base_dir. -- Catches ValueError on invalid path resolution. ---- - app.py | 12 +++++++++--- - 1 file changed, 9 insertions(+), 3 deletions(-) - -diff --git a/app.py b/app.py -index a1b2c3d..e4f5g6h 100644 ---- a/app.py -+++ b/app.py -@@ -15,13 +15,19 @@ def serve_report(request_payload): - # Helper function nested inside serve_report - def build_file_path(filename): -- # VULNERABLE: Direct concatenation allows Path Traversal -- # An attacker passing "../../etc/passwd" could read system files. -- return os.path.join(generator.base_dir, filename) -+ # FIXED: Validate that the resolved path stays within the base_dir -+ base = os.path.abspath(generator.base_dir) -+ target = os.path.abspath(os.path.join(base, filename)) -+ if not target.startswith(base): -+ raise ValueError("Path Traversal Detected") -+ return target - - if not requested_file: - return "Error: No file specified" - -- target_path = build_file_path(requested_file) -+ try: -+ target_path = build_file_path(requested_file) -+ except ValueError: -+ return "Error: Invalid path" - - if os.path.exists(target_path): - return f"Serving content of {target_path}" \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/fixed-app.py index ca5a6f4c8b..30470b6cda 100644 --- a/scanpipe/tests/data/reachability/fixed-app.py +++ b/scanpipe/tests/data/reachability/fixed-app.py @@ -1,5 +1,7 @@ import os +debug = True + class ReportGenerator: """A dummy class to test AST class method parsing.""" diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/vuln-app.py index b8c9eff5e0..32eebc2fa1 100644 --- a/scanpipe/tests/data/reachability/vuln-app.py +++ b/scanpipe/tests/data/reachability/vuln-app.py @@ -1,5 +1,7 @@ import os +debug = False + class ReportGenerator: """A dummy class to test AST class method parsing.""" diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index e24c42932b..0d3ff42fa1 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -55,16 +55,16 @@ def test_get_symbol_reachability_results( mock_collect_symbols, mock_git_context, ): - app_text = (self.data / "app.py").read_text() + file_path = "app.py" + app_text = (self.data / file_path).read_text() vuln_text = (self.data / "vuln-app.py").read_text() fixed_text = (self.data / "fixed-app.py").read_text() - diff_text = (self.data / "diff-app.patch").read_text() analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") - diff_line_map = analyzer.parse_diff_lines(diff_text=diff_text) - file_path = "app.py" - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + removed_lines, added_lines = analyzer.compute_changed_lines( + vulnerable_text=vuln_text, fixed_text=fixed_text + ) vuln_meta, fixed_meta, lang = analyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, @@ -90,10 +90,12 @@ def test_get_symbol_reachability_results( mock_collect_symbols.return_value = { lang: { "vulnerable": { - f"app.py::{key}": metadata for key, metadata in vuln_meta.items() + f"{file_path}::{key}": metadata + for key, metadata in vuln_meta.items() }, "fixed": { - f"app.py::{key}": metadata for key, metadata in fixed_meta.items() + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() }, } } @@ -125,9 +127,9 @@ def test_get_symbol_reachability_results( "called": False, "defined": True, "imported": False, - "fingerprint": "d7675efb263896da2a3c00679511833" - "553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", + "fingerprint": "336908735214468b103dbde" + "11c3ffbd2f76ac9212b8514f831cfa078a67892df", + "symbol_name": "debug", "reachable_from": [], }, { @@ -139,12 +141,23 @@ def test_get_symbol_reachability_results( "symbol_name": "serve_report.build_file_path", "reachable_from": ["serve_report"], }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, ], "fixed_symbols": [ + "debug", "serve_report", "serve_report.build_file_path", ], "vulnerable_symbols": [ + "debug", "serve_report", "serve_report.build_file_path", ], @@ -402,59 +415,68 @@ def test_diff_changed_symbols(self): def test_analyze_patched_file(self): vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") - diff_text = (self.data / "diff-app.patch").read_text(encoding="utf-8") - - diff_line_map = PatchAnalyzer.parse_diff_lines(diff_text=diff_text) file_path = "app.py" - removed_lines, added_lines = diff_line_map.get(file_path, ([], [])) + removed_lines, added_lines = PatchAnalyzer.compute_changed_lines( + vuln_text, fixed_text + ) vuln_meta, fixed_meta, lang = PatchAnalyzer.analyze( vulnerable_text=vuln_text, fixed_text=fixed_text, removed_lines=removed_lines, added_lines=added_lines, - file_path="app.py", + file_path=file_path, ) self.assertEqual( vuln_meta, { + "debug": { + "qualified_name": "debug", + "text": "debug = False", + "fingerprint": "336908735214468b103dbde11c3ff" + "bd2f76ac9212b8514f831cfa078a67892df", + "start_line": 3, + "end_line": 3, + "node_type": "assignment", + }, + "serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": "def build_file_path(filename):\n" + " # VULNERABLE: Direct concatenation " + "allows Path Traversal\n " + ' # An attacker passing "../../etc/passwd" ' + "could read system files.\n" + " return os.path.join(generator.base_dir, filename)", + "fingerprint": "762e4f7d03b1bf4359c3ca364e55814" + "0239913bfabcc5aa77156460c2eb0a355", + "start_line": 19, + "end_line": 22, + "node_type": "function_definition", + }, "serve_report": { "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n" - ' """Top-level function handling a request."""\n' + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n' ' generator = ReportGenerator("/var/reports")\n' ' requested_file = request_payload.get("file")\n\n' " # Helper function nested inside serve_report\n" - " def build_file_path(filename):\n" - " # VULNERABLE: Direct concatenation allows Path Traversal\n" - " # An attacker passing " - '"../../etc/passwd" ' - "could read system files.\n" + " def build_file_path(filename):\n " + " # VULNERABLE: Direct " + "concatenation allows Path Traversal\n " + " # An attacker passing " + '"../../etc/passwd" could read system files.\n' " return os.path.join(generator.base_dir, filename)\n\n" - " if not requested_file:\n" - ' return "Error: No file specified"\n\n' - " target_path = build_file_path(requested_file)\n\n" - " if os.path.exists(target_path):\n" - ' return f"Serving content of {target_path}"\n\n' - ' return "Error: File not found"', + " if not requested_file:\n " + ' return "Error: No file specified"\n\n ' + " target_path = build_file_path(requested_file)\n\n " + " if os.path.exists(target_path):\n " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', "fingerprint": "d7675efb263896da2a3c006795118" "33553907e7e6ea619115a6dfc8625c3457e", - "start_line": 11, - "end_line": 30, - "node_type": "function_definition", - }, - "serve_report.build_file_path": { - "qualified_name": "serve_report.build_file_path", - "text": "def build_file_path(filename):\n" - " # VULNERABLE: Direct concatenation allows Path Traversal\n" - ' # An attacker passing "../../etc/passwd"' - " could read system files.\n" - " return os.path.join(generator.base_dir, filename)", - "fingerprint": "762e4f7d03b1bf4359c3ca364e5581" - "40239913bfabcc5aa77156460c2eb0a355", - "start_line": 17, - "end_line": 20, + "start_line": 13, + "end_line": 32, "node_type": "function_definition", }, }, @@ -463,50 +485,58 @@ def test_analyze_patched_file(self): self.assertEqual( fixed_meta, { - "serve_report": { - "qualified_name": "serve_report", - "text": "def serve_report(request_payload):\n" - ' """Top-level function handling a request."""\n' - ' generator = ReportGenerator("/var/reports")\n' - ' requested_file = request_payload.get("file")\n\n' - " # Helper function nested inside serve_report\n" - " def build_file_path(filename):\n" - " # FIXED: Validate that the resolved " - "path stays within the base_dir\n" - " base = os.path.abspath(generator.base_dir)\n" - " target = os.path.abspath(os.path.join(base, filename))\n" - " if not target.startswith(base):\n" - ' raise ValueError("Path Traversal Detected")\n' - " return target\n\n" - " if not requested_file:\n" - ' return "Error: No file specified"\n\n' - " try:\n" - " target_path = build_file_path(requested_file)\n" - " except ValueError:\n" - ' return "Error: Invalid path"\n\n' - " if os.path.exists(target_path):\n" - ' return f"Serving content of {target_path}"\n\n' - ' return "Error: File not found"', - "fingerprint": "2deedb21d5f9b1409c59f0b1e5512d" - "73d9afdfc3f469ccf86e8835915d240e76", - "start_line": 11, - "end_line": 36, - "node_type": "function_definition", + "debug": { + "qualified_name": "debug", + "text": "debug = True", + "fingerprint": "55d2e2010de610fd32f0c28bc49f535" + "3d6ac60afc70adc5713aa4b675646590e", + "start_line": 3, + "end_line": 3, + "node_type": "assignment", }, "serve_report.build_file_path": { "qualified_name": "serve_report.build_file_path", - "text": "def build_file_path(filename):\n" - " # FIXED: Validate that the resolved path" - " stays within the base_dir\n" - " base = os.path.abspath(generator.base_dir)\n" - " target = os.path.abspath(os.path.join(base, filename))\n" - " if not target.startswith(base):\n" - ' raise ValueError("Path Traversal Detected")\n' - " return target", + "text": "def build_file_path(filename):\n " + " # FIXED: Validate that the resolved" + " path stays within the base_dir\n " + " base = os.path.abspath(generator.base_dir)\n " + " target = os.path.abspath(os.path.join(base, filename))\n " + " if not target.startswith(base):\n " + ' raise ValueError("Path Traversal Detected")\n ' + " return target", "fingerprint": "646743b5d5497f6ea3b96f860bcbe" "b38096ce008ad16d2b9a9c3f77a98faca80", - "start_line": 17, - "end_line": 23, + "start_line": 19, + "end_line": 25, + "node_type": "function_definition", + }, + "serve_report": { + "qualified_name": "serve_report", + "text": "def serve_report(request_payload):\n " + ' """Top-level function handling a request."""\n ' + ' generator = ReportGenerator("/var/reports")\n ' + ' requested_file = request_payload.get("file")\n\n ' + " # Helper function nested inside serve_report\n " + "def build_file_path(filename):\n " + " # FIXED: Validate that the" + " resolved path stays within the base_dir\n " + " base = os.path.abspath(generator.base_dir)\n " + " target = os.path.abspath(os.path.join(base, filename))\n" + " if not target.startswith(base):\n " + ' raise ValueError("Path Traversal Detected")\n ' + " return target\n\n " + " if not requested_file:\n " + ' return "Error: No file specified"\n\n try:\n ' + " target_path = build_file_path(requested_file)\n" + " except ValueError:\n " + ' return "Error: Invalid path"\n\n ' + " if os.path.exists(target_path):\n " + ' return f"Serving content of {target_path}"\n\n ' + ' return "Error: File not found"', + "fingerprint": "2deedb21d5f9b1409c59f0b1e55" + "12d73d9afdfc3f469ccf86e8835915d240e76", + "start_line": 13, + "end_line": 38, "node_type": "function_definition", }, }, From 0d85586b4e361e4eda9c54d4a4cccf6fc3ee417c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 7 Aug 2026 21:42:42 +0300 Subject: [PATCH 26/50] Add missing docs Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 42 ++++- scanpipe/pipes/reachability.py | 148 ++++++++++++++++-- .../tests/pipes/test_symbols_reachability.py | 61 ++++---- 3 files changed, 202 insertions(+), 49 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 2489df9c49..f0939eb179 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -1,18 +1,46 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. # SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. # +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + from scanpipe.pipelines import Pipeline from scanpipe.pipes import reachability class SymbolReachability(Pipeline): - """Patch reachability analysis for given vulnerability patches.""" + """ + Patch reachability analysis for given vulnerability patches. + + For every patch we clone the git repository and extract + the vulnerable and fixed symbols from the patch commit. + + These symbols are then matched against the project's codebase resources to + determine if the vulnerable code is actually present and reachable. + + The analysis checks if vulnerable symbols are defined, imported, called, or + exactly match a fingerprint within the project files. The results, including + evidence and a reachability status (REACHABLE, POTENTIALLY_REACHABLE, or + NOT_REACHABLE), are stored in the `extra_data` of the matching resources + under the `symbols_reachability` key. + """ download_inputs = False is_addon = True diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 6c7768dbad..39ded984b1 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -26,7 +26,6 @@ import tempfile from enum import Enum from pathlib import Path -from typing import Any from git import Repo from git.diff import NULL_TREE @@ -45,6 +44,7 @@ class ReachabilityStatus(str, Enum): def normalize_text(content): + """Normalize content (bytes) into a UTF-8 decoded string.""" if content is None: return "" @@ -55,6 +55,10 @@ def normalize_text(content): def detect_language_with_scancode(file_path, content): + """ + Detect the programming language of the text + content using get_file_info function + """ content = normalize_text(content) if not content: @@ -74,12 +78,12 @@ def detect_language_with_scancode(file_path, content): class GitRepositoryContext: - def __init__(self, vcs_url: str): + def __init__(self, vcs_url): self.vcs_url = vcs_url self.repo_path = None self._repo = None - def __enter__(self) -> "GitRepositoryContext": + def __enter__(self): self.repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") try: self._repo = Repo.clone_from(self.vcs_url, self.repo_path) @@ -101,12 +105,25 @@ def _cleanup(self): class PatchAnalyzer: - def __init__(self, repo: Repo, commit_hash: str): + def __init__(self, repo: Repo, commit_hash): self.repo = repo self.commit = repo.commit(commit_hash) self.parent_commit = self.commit.parents[0] if self.commit.parents else None def get_changed_files(self): + """ + Retrieve all files changed by the commit along with their + vulnerable and fixed contents. + + For each changed file, a dictionary entry is created with two + keys: + + - vulnerable_text: The file content before the commit (empty + string for newly added files). + - fixed_text: The file content after the commit (empty string + for deleted files). + + """ diffs = ( self.parent_commit.diff(self.commit, create_patch=False) if self.parent_commit @@ -170,6 +187,15 @@ def compute_changed_lines(cls, vulnerable_text, fixed_text): @classmethod def diff_changed_symbols(cls, vuln_meta, fixed_meta): + """ + Compare the vulnerable and fixed symbol metadata and return the + symbols that are unique to each side (i.e., whose body text + differs between the two versions). + + A symbol key is considered "vulnerable-only" if its body text + does not match the corresponding symbol in fixed_meta, and + vice versa. + """ vuln_only = { key: metadata for key, metadata in vuln_meta.items() @@ -183,6 +209,19 @@ def diff_changed_symbols(cls, vuln_meta, fixed_meta): return vuln_only, fixed_only def collect_patch_symbols(self): + """ + Collect all changed symbols across every file modified by the + commit, grouped by programming language. + + For each changed file, the analyzer: + - Retrieves the vulnerable and fixed file contents. + - Computes which lines were removed and added. + - Extracts symbols that intersect those changed lines using + Tree-sitter parsing SymbolExtractor + - Diffs the extracted symbols to find those whose body text + actually changed. + - Buckets the results by programming language. + """ by_language = {} changed_files = self.get_changed_files() @@ -221,9 +260,16 @@ def collect_patch_symbols(self): return by_language @classmethod - def build_symbol_metadata( - cls, nodes, extractor: SymbolExtractor, index: dict = None - ): + def build_symbol_metadata(cls, nodes, extractor: SymbolExtractor, index): + """ + Build metadata dictionaries for a list of Tree-sitter AST nodes + representing changed symbols. + + For each node, the qualified name, body text, SHA-256 fingerprint, + and start/end line numbers are extracted and stored in a + dictionary keyed by the qualified name. If duplicate qualified + names are encountered, a numeric suffix is appended to disambiguate. + """ if not nodes or not extractor: return {} @@ -259,6 +305,23 @@ def build_symbol_metadata( def analyze( cls, vulnerable_text, fixed_text, removed_lines, added_lines, file_path ): + """ + Analyze the vulnerable and fixed versions of a single file to + extract changed symbols. + + The method performs the following steps: + + - Detects the programming language of the file (using the + fixed version first, falling back to the vulnerable version). + - Verifies the language is supported by Tree-sitter queries. + - Parses both versions into ASTs using Tree-sitter. + - Extracts symbols whose line ranges intersect the removed + lines (vulnerable side) or added lines (fixed side). + - Builds metadata for each set of changed symbols. + - Diffs the two metadata sets to find symbols whose body text + actually changed between versions. + + """ vulnerable_text = normalize_text(vulnerable_text) fixed_text = normalize_text(fixed_text) @@ -317,6 +380,23 @@ def analyze( def generate_reachability_report(patch, repo, candidate_resources, logger=None): + """ + Generate and store a symbol reachability report for a single + vulnerability patch against a set of candidate codebase resources. + + - Creates a PatchAnalyzer for the given commit. + - Collects changed symbols from the patch, grouped by language. + - For each candidate resource whose language matches a patch + language, builds a ResourceAnalyzer index and uses + ResourcePatchMatcher to match vulnerable and fixed + symbols. + - If matches are found, constructs a report dict containing + evidence, matched symbol names, and a ReachabilityStatus. + - Appends the report to the resource's extra_data under the + symbols_reachability key (avoiding duplicates for the same + commit hash). + + """ vcs_url = patch.get("vcs_url") commit_hash = patch.get("commit_hash") @@ -384,6 +464,20 @@ def generate_reachability_report(patch, repo, candidate_resources, logger=None): def analyze_and_store_symbol_reachability_results(project, logger=None): + """ + Analyze all vulnerability patches for a project and store the + resulting symbol reachability reports + + The function iterates over all package vulnerabilities associated + with the project, groups their patches by repository URL, clones + each repository (using :class:`GitRepositoryContext`), checks out + each patch's commit, and calls :func:`generate_reachability_report` + to determine whether the vulnerable/fixed symbols from the patch + are reachable in the project's codebase resources. + + Only non-binary, non-archive, non-media files are considered as + candidate resources. + """ candidate_resources = project.codebaseresources.files().filter( is_binary=False, is_archive=False, is_media=False ) @@ -414,6 +508,10 @@ def analyze_and_store_symbol_reachability_results(project, logger=None): def classify_reachability(evidence): + """ + Classify the reachability status of a vulnerability based on the + collected evidence from :class:`ResourcePatchMatcher`. + """ if not evidence: return ReachabilityStatus.NOT_REACHABLE @@ -435,14 +533,18 @@ def classify_reachability(evidence): class ResourceAnalyzer: - def __init__(self, resource_text: str, language: str): + def __init__(self, resource_text, language): self.resource_text = normalize_text(resource_text) self.language = language def process_node( - self, node, extractor, definitions_index, definitions: set, fingerprints: set - ) -> str | None: - """Extract the qualified name, update definitions, and fingerprint""" + self, node, extractor, definitions_index, definitions, fingerprints + ): + """ + Process a single AST node to extract its qualified name, add it + to the definitions set, compute its fingerprint, and add the + fingerprint to the fingerprints set. + """ qualified_name = extractor._build_qualified_name(node, definitions_index) if not qualified_name: return None @@ -456,7 +558,16 @@ def process_node( return qualified_name - def build_index(self) -> dict | None: + def build_index(self): + """ + Build the full symbol index for the resource by parsing it + with Tree-sitter and extracting all definitions, fingerprints, + imports, and the reverse call graph. + + The method iterates over all functions, classes, and constants + in the resource's AST. For functions, it also extracts call + expressions to populate the callers_of reverse call graph. + """ if not is_supported_language(self.language) or not self.resource_text: return None @@ -503,7 +614,7 @@ def build_index(self) -> dict | None: class ResourcePatchMatcher: - def __init__(self, resource_index: dict): + def __init__(self, resource_index): self.resource_index = resource_index self.definitions = resource_index.get("definitions", set()) self.fingerprints = resource_index.get("fingerprints", set()) @@ -511,7 +622,16 @@ def __init__(self, resource_index: dict): self.callers_of = resource_index.get("callers_of", {}) self.separator = resource_index.get("separator", ".") - def match(self, patch_symbols_metadata: dict[str, Any]) -> dict[str, Any]: + def match(self, patch_symbols_metadata): + """ + Match a set of patch symbols against the resource index and + return evidence for each matched symbol. + + For each symbol in patch_symbols_metadata, the method + checks whether it is defined, imported, called, or has an + exact fingerprint match in the resource. If at least one of + these conditions is true, an evidence entry is created. + """ if not patch_symbols_metadata or not self.resource_index: return {} diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 0d3ff42fa1..9a15773a44 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -55,6 +55,11 @@ def test_get_symbol_reachability_results( mock_collect_symbols, mock_git_context, ): + """ + Test the end-to-end reachability pipeline by analyzing a patch, + computing symbol reachability, and storing the results on the + corresponding codebase resource. + """ file_path = "app.py" app_text = (self.data / file_path).read_text() vuln_text = (self.data / "vuln-app.py").read_text() @@ -168,6 +173,10 @@ def test_get_symbol_reachability_results( ) def test_extract_definitions(self): + """ + Test extracting functions, classes, and constant + definitions from Python code. + """ source_code = """ price = 0 class OrderManager: @@ -205,6 +214,7 @@ class InventoryItem: self.assertEqual(len(constants), 1) def test_extract_definitions_empty(self): + """Test parsing empty or None source code""" lang_query = TS_QUERIES["Python"]() tree, _ = lang_query.parse_code_to_ast("") @@ -214,6 +224,7 @@ def test_extract_definitions_empty(self): self.assertIsNone(tree_none) def test_get_qualified_name_functions(self): + """Test building qualified names for nested and top-level functions.""" source_code = """ class CoreService: class Validator: @@ -240,6 +251,7 @@ def global_utility(): self.assertEqual(inner_function_name, "global_utility") def test_get_qualified_classes(self): + """Test building qualified names for nested class definitions.""" source_code = """ class FleetManagement: class DroneController: @@ -261,6 +273,10 @@ class DroneController: self.assertEqual(inner_class_name, "FleetManagement.DroneController") def test_classify_reachability(self): + """ + Test reachability classification for fingerprint + and evidence-based results. + """ self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) self.assertEqual( @@ -285,6 +301,7 @@ def test_classify_reachability(self): ) def test_build_symbol_metadata_processing(self): + """Test building metadata for nested changed symbols with deduplication.""" source_code = """ class Controller: def process_data(payload): @@ -347,6 +364,7 @@ def process_data(payload): ) def test_diff_changed_symbols(self): + """Test that changed, added, and removed symbols are correctly identified.""" vuln_meta = { "serve_report": { "qualified_name": "app.serve_report", @@ -413,6 +431,7 @@ def test_diff_changed_symbols(self): ) def test_analyze_patched_file(self): + """Test analyzing a patched file and extracting changed symbol metadata.""" vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") file_path = "app.py" @@ -543,6 +562,7 @@ def test_analyze_patched_file(self): ) def test_extract_symbols(self): + """Test extracting only the nested function containing the changed line.""" source_code = ( "def serve_report(request):\n" # Line 1 (Row 0) " # Some processing here\n" # Line 2 (Row 1) @@ -567,6 +587,10 @@ def test_extract_symbols(self): self.assertNotIn("def serve_report", node_text) def test_extract_symbols_deduplication(self): + """ + Test that multiple changed lines within + the same function produce only a single enclosing symbol. + """ source_code = ( "def calculate_total(price, tax):\n" " amount = price * tax\n" # Line 2 -> Changed @@ -584,35 +608,11 @@ def test_extract_symbols_deduplication(self): self.assertEqual(len(enclosing_symbols), 1) self.assertEqual(enclosing_symbols[0].type, "function_definition") - def test_collect_imports(self): - source_code = """ -from django.db import models -import os.path -import numpy as np -from a.b import c as d -from . import utils -from ..core import engine -from math import * - """.strip() - - lang_query = TS_QUERIES["Python"]() - tree, _ = lang_query.parse_code_to_ast(code_text=source_code) - extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) - result = extractor.extract_imports() - - expected_map = { - "models": "django.db.models", - "os": "os.path", - "np": "numpy", - "d": "a.b.c", - "utils": ".utils", - "engine": "..core.engine", - "*": ["math"], - } - - self.assertEqual(result, expected_map) - def test_extract_direct(self): + """ + Test that direct function calls are + extracted from the syntax tree. + """ source_code = """ def hello(): return 10 @@ -632,6 +632,7 @@ def clean_function(): ) def test_extract_direct_calls(self): + """Test extraction of direct function calls.""" python_source = """ self.update() process_data() @@ -651,6 +652,10 @@ def test_extract_direct_calls(self): self.assertEqual(expected_python, python_calls) def test_extract_imports(self): + """ + Test extraction of imported symbols and their + fully qualified module paths. + """ source_code = """ from django.db import models import os.path From c9414eac5731466673669032e67c9877c58dc7f3 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 7 Aug 2026 23:47:42 +0300 Subject: [PATCH 27/50] Fix a typo in build_symbol_metadata function signature Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 7 +++---- scanpipe/tests/pipes/test_symbols_reachability.py | 3 +-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 39ded984b1..6bdd8839ab 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -260,7 +260,7 @@ def collect_patch_symbols(self): return by_language @classmethod - def build_symbol_metadata(cls, nodes, extractor: SymbolExtractor, index): + def build_symbol_metadata(cls, nodes, extractor): """ Build metadata dictionaries for a list of Tree-sitter AST nodes representing changed symbols. @@ -273,8 +273,7 @@ def build_symbol_metadata(cls, nodes, extractor: SymbolExtractor, index): if not nodes or not extractor: return {} - if index is None: - index = extractor.extract_definitions_index() + index = extractor.extract_definitions_index() metadata = {} for node in nodes: @@ -370,7 +369,7 @@ def analyze( changed_lines=added_lines ) fixed_meta_all = cls.build_symbol_metadata( - fixed_nodes, extractor=fixed_extractor + nodes=fixed_nodes, extractor=fixed_extractor ) vuln_meta, fixed_meta = cls.diff_changed_symbols( diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 9a15773a44..b1557f647e 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -316,12 +316,11 @@ def process_data(payload): lang_query = TS_QUERIES["Python"]() tree, _ = lang_query.parse_code_to_ast(source_code) extractor = SymbolExtractor(lang_query=lang_query, root_node=tree.root_node) - index = extractor.extract_definitions_index() vuln_nodes = extractor.extract_changed_symbols( changed_lines=[1, 2, 3, 4, 5, 6, 7, 8, 9] ) metadata = PatchAnalyzer.build_symbol_metadata( - nodes=vuln_nodes, extractor=extractor, index=index + nodes=vuln_nodes, extractor=extractor ) self.assertEqual( metadata, From 566ae1e9972c33e2fb93dfecf3751faa83d01a79 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 10 Aug 2026 13:44:21 +0300 Subject: [PATCH 28/50] Add a test for java Add end-to-end test for Symbol Reachability pipeline Signed-off-by: ziad hany --- .../tests/data/reachability/java/app.java | 50 ++++ .../data/reachability/java/fixed-app.java | 60 +++++ .../data/reachability/java/vuln-app.java | 46 ++++ .../data/reachability/{ => python}/app.py | 0 .../reachability/{ => python}/fixed-app.py | 0 .../reachability/{ => python}/vuln-app.py | 0 .../tests/pipes/test_symbols_reachability.py | 232 +++++++++++++++++- 7 files changed, 381 insertions(+), 7 deletions(-) create mode 100644 scanpipe/tests/data/reachability/java/app.java create mode 100644 scanpipe/tests/data/reachability/java/fixed-app.java create mode 100644 scanpipe/tests/data/reachability/java/vuln-app.java rename scanpipe/tests/data/reachability/{ => python}/app.py (100%) rename scanpipe/tests/data/reachability/{ => python}/fixed-app.py (100%) rename scanpipe/tests/data/reachability/{ => python}/vuln-app.py (100%) diff --git a/scanpipe/tests/data/reachability/java/app.java b/scanpipe/tests/data/reachability/java/app.java new file mode 100644 index 0000000000..0e0b52b178 --- /dev/null +++ b/scanpipe/tests/data/reachability/java/app.java @@ -0,0 +1,50 @@ +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; + +public class App { + public static boolean debug = true; + + public static class ReportGenerator { + private String baseDir; + + public ReportGenerator(String baseDir) { + this.baseDir = baseDir; + } + + public String getBaseDir() { + return baseDir; + } + } + + public static String serveReport(Map requestPayload) { + ReportGenerator generator = new ReportGenerator("/var/reports"); + String requestedFile = requestPayload.get("file"); + + if (requestedFile == null || requestedFile.isEmpty()) { + return "Error: No file specified"; + } + + // VULNERABLE: Direct concatenation allows Path Traversal + // An attacker passing "../../etc/passwd" could read system files. + String targetPath = buildFilePath(generator, requestedFile); + + try { + if (Files.exists(Paths.get(targetPath))) { + return "Serving content of " + targetPath; + } + } catch (Exception e) { + return "Error: Invalid path"; + } + + return "Error: File not found"; + } + + private static String buildFilePath(ReportGenerator generator, String filename) { + return Paths.get(generator.getBaseDir(), filename).toString(); + } + + public static String unrelatedTopLevelFunction() { + return "I am just here to add AST complexity."; + } +} \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/java/fixed-app.java b/scanpipe/tests/data/reachability/java/fixed-app.java new file mode 100644 index 0000000000..39234131b0 --- /dev/null +++ b/scanpipe/tests/data/reachability/java/fixed-app.java @@ -0,0 +1,60 @@ +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +public class App { + public static boolean debug = false; + + public static class ReportGenerator { + public String baseDir; + + public ReportGenerator(String baseDir) { + this.baseDir = baseDir; + } + } + + public static String serveReport(Map requestPayload) { + ReportGenerator generator = new ReportGenerator("/var/reports"); + String requestedFile = requestPayload.get("file"); + + if (requestedFile == null || requestedFile.isEmpty()) { + return "Error: No file specified"; + } + + String targetPath; + try { + targetPath = buildFilePath(generator, requestedFile); + } catch (Exception e) { + return "Error: Invalid path"; + } + + try { + if (Files.exists(Paths.get(targetPath))) { + return "Serving content of " + targetPath; + } + } catch (Exception e) { + return "Error: Invalid path"; + } + + return "Error: File not found"; + } + + /** + * FIXED: Validate that the resolved path stays within the base_dir + */ + private static String buildFilePath(ReportGenerator generator, String filename) throws Exception { + Path base = Paths.get(generator.baseDir).toAbsolutePath().normalize(); + Path target = Paths.get(generator.baseDir, filename).toAbsolutePath().normalize(); + + if (!target.startsWith(base)) { + throw new Exception("Path Traversal Detected"); + } + return target.toString(); + } + + + public static String unrelatedTopLevelFunction() { + return "I am just here to add AST complexity."; + } +} \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/java/vuln-app.java b/scanpipe/tests/data/reachability/java/vuln-app.java new file mode 100644 index 0000000000..548ffd9eab --- /dev/null +++ b/scanpipe/tests/data/reachability/java/vuln-app.java @@ -0,0 +1,46 @@ +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.Map; + +public class App { + public static boolean debug = true; + + public static class ReportGenerator { + public String baseDir; + + public ReportGenerator(String baseDir) { + this.baseDir = baseDir; + } + } + + public static String serveReport(Map requestPayload) { + ReportGenerator generator = new ReportGenerator("/var/reports"); + String requestedFile = requestPayload.get("file"); + + if (requestedFile == null || requestedFile.isEmpty()) { + return "Error: No file specified"; + } + + // VULNERABLE: Direct concatenation allows Path Traversal + // An attacker passing "../../etc/passwd" could read system files. + String targetPath = buildFilePath(generator, requestedFile); + + try { + if (Files.exists(Paths.get(targetPath))) { + return "Serving content of " + targetPath; + } + } catch (Exception e) { + return "Error: Invalid path"; + } + + return "Error: File not found"; + } + + private static String buildFilePath(ReportGenerator generator, String filename) { + return Paths.get(generator.baseDir, filename).toString(); + } + + public static String unrelatedTopLevelFunction() { + return "I am just here to add AST complexity."; + } +} \ No newline at end of file diff --git a/scanpipe/tests/data/reachability/app.py b/scanpipe/tests/data/reachability/python/app.py similarity index 100% rename from scanpipe/tests/data/reachability/app.py rename to scanpipe/tests/data/reachability/python/app.py diff --git a/scanpipe/tests/data/reachability/fixed-app.py b/scanpipe/tests/data/reachability/python/fixed-app.py similarity index 100% rename from scanpipe/tests/data/reachability/fixed-app.py rename to scanpipe/tests/data/reachability/python/fixed-app.py diff --git a/scanpipe/tests/data/reachability/vuln-app.py b/scanpipe/tests/data/reachability/python/vuln-app.py similarity index 100% rename from scanpipe/tests/data/reachability/vuln-app.py rename to scanpipe/tests/data/reachability/python/vuln-app.py diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index b1557f647e..72038e2d6c 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -19,7 +19,9 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/nexB/scancode.io for support and download. +import shutil import sys +import tempfile from pathlib import Path from unittest import skipIf from unittest.mock import MagicMock @@ -46,10 +48,101 @@ def setUp(self): self.project1 = Project.objects.create(name="Analysis") self.project1.codebase_path.mkdir(parents=True, exist_ok=True) + @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) + def test_end_to_end_symbol_reachability_pipeline( + self, mock_package_vulnerabilities + ): + """ + Global end-to-end test for the symbol reachability pipeline. + Sets up a local git repository with a vulnerable and fixed commit, + then runs the full pipeline against a local codebase resource. + """ + from git import Repo + + vcs_dir = tempfile.mkdtemp(prefix="test-vcs-") + try: + repo = Repo.init(vcs_dir) + with repo.config_writer() as config: + config.set_value("user", "email", "test@example.com") + config.set_value("user", "name", "test") + + file_path = "app.py" + vuln_code = ( + "def process_data(data):\n" + " # Vulnerable logic\n" + " return eval(data)\n" + ) + fixed_code = ( + "def process_data(data):\n # Fixed logic\n return int(data)\n" + ) + + vuln_file = Path(vcs_dir) / file_path + vuln_file.write_text(vuln_code) + repo.index.add([file_path]) + repo.index.commit("Vulnerable commit") + + vuln_file.write_text(fixed_code) + repo.index.add([file_path]) + fixed_commit = repo.index.commit("Fixed commit") + + resource_file = self.project1.codebase_path / "local_app.py" + resource_file.write_text(vuln_code) + collect_and_create_codebase_resources(self.project1) + + resource = self.project1.codebaseresources.get(path="local_app.py") + resource.programming_language = "Python" + resource.save() + + repo_url = Path(vcs_dir).as_uri() + + mock_package_vulnerabilities.return_value = [ + { + "fixed_in_patches": [ + { + "vcs_url": repo_url, + "commit_hash": fixed_commit.hexsha, + } + ] + } + ] + + analyze_and_store_symbol_reachability_results(self.project1) + resource.refresh_from_db() + results = resource.extra_data.get("symbols_reachability") + + expected_results = [ + { + "symbols_reachability": { + "patch": { + "vcs_url": repo_url, + "commit_hash": fixed_commit.hexsha, + }, + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "e66988810af452f77d43efd302fea4c" + "1d2f246d282c4ea8982b0152c2231ac17", + "symbol_name": "process_data", + "reachable_from": [], + } + ], + "fixed_symbols": ["process_data"], + "vulnerable_symbols": ["process_data"], + "reachability_status": "REACHABLE", + } + } + ] + + self.assertEqual(results, expected_results) + finally: + shutil.rmtree(vcs_dir, ignore_errors=True) + @patch("scanpipe.pipes.reachability.GitRepositoryContext") @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) - def test_get_symbol_reachability_results( + def test_python_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, @@ -61,9 +154,9 @@ def test_get_symbol_reachability_results( corresponding codebase resource. """ file_path = "app.py" - app_text = (self.data / file_path).read_text() - vuln_text = (self.data / "vuln-app.py").read_text() - fixed_text = (self.data / "fixed-app.py").read_text() + app_text = (self.data / "python" / file_path).read_text() + vuln_text = (self.data / "python" / "vuln-app.py").read_text() + fixed_text = (self.data / "python" / "fixed-app.py").read_text() analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") @@ -172,6 +265,131 @@ def test_get_symbol_reachability_results( ], ) + @patch("scanpipe.pipes.reachability.GitRepositoryContext") + @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") + @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) + def test_java_get_symbol_reachability_results( + self, + mock_package_vulnerabilities, + mock_collect_symbols, + mock_git_context, + ): + """ + Test the end-to-end reachability pipeline by analyzing a patch, + computing symbol reachability, and storing the results on the + corresponding codebase resource. + """ + file_path = "app.java" + app_text = (self.data / "java" / file_path).read_text() + vuln_text = (self.data / "java" / "vuln-app.java").read_text() + fixed_text = (self.data / "java" / "fixed-app.java").read_text() + + analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") + + removed_lines, added_lines = analyzer.compute_changed_lines( + vulnerable_text=vuln_text, fixed_text=fixed_text + ) + vuln_meta, fixed_meta, lang = analyzer.analyze( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, + ) + + self.assertTrue(lang) + self.assertTrue(vuln_meta or fixed_meta) + mock_package_vulnerabilities.return_value = [ + { + "fixed_in_patches": [ + { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + } + ] + } + ] + + mock_git_context.return_value.__enter__.return_value.repo = MagicMock() + mock_collect_symbols.return_value = { + lang: { + "vulnerable": { + f"{file_path}::{key}": metadata + for key, metadata in vuln_meta.items() + }, + "fixed": { + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() + }, + } + } + + resource_file = self.project1.codebase_path / "app.java" + resource_file.parent.mkdir(parents=True, exist_ok=True) + resource_file.write_text(app_text) + collect_and_create_codebase_resources(self.project1) + + resource = self.project1.codebaseresources.get(path="app.java") + resource.programming_language = lang + resource.save() + + analyze_and_store_symbol_reachability_results(self.project1) + + resource.refresh_from_db() + results = resource.extra_data.get("symbols_reachability") + + self.assertEqual( + results, + [ + { + "symbols_reachability": { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": None, + "symbol_name": "App", + "reachable_from": [], + }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0" + "a1e6f19e2c4baa421a6cc996fda154", + "symbol_name": "App.serveReport", + "reachable_from": [], + }, + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": None, + "symbol_name": "App.buildFilePath", + "reachable_from": ["App.serveReport"], + }, + ], + "fixed_symbols": [ + "App", + "App.buildFilePath", + "App.serveReport", + ], + "vulnerable_symbols": [ + "App", + "App.buildFilePath", + "App.serveReport", + ], + "reachability_status": "REACHABLE", + } + } + ], + ) + def test_extract_definitions(self): """ Test extracting functions, classes, and constant @@ -431,9 +649,9 @@ def test_diff_changed_symbols(self): def test_analyze_patched_file(self): """Test analyzing a patched file and extracting changed symbol metadata.""" - vuln_text = (self.data / "vuln-app.py").read_text(encoding="utf-8") - fixed_text = (self.data / "fixed-app.py").read_text(encoding="utf-8") - file_path = "app.py" + vuln_text = (self.data / "python" / "vuln-app.py").read_text(encoding="utf-8") + fixed_text = (self.data / "python" / "fixed-app.py").read_text(encoding="utf-8") + file_path = "python/app.py" removed_lines, added_lines = PatchAnalyzer.compute_changed_lines( vuln_text, fixed_text ) From 9985d215428b6cda794a448ded968585cfb39951 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 11 Aug 2026 03:47:52 +0300 Subject: [PATCH 29/50] Simplify the pipeline test Signed-off-by: ziad hany --- .../tests/pipes/test_symbols_reachability.py | 307 ++++++++---------- 1 file changed, 138 insertions(+), 169 deletions(-) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 72038e2d6c..4cc0ad74cf 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -139,25 +139,18 @@ def test_end_to_end_symbol_reachability_pipeline( finally: shutil.rmtree(vcs_dir, ignore_errors=True) - @patch("scanpipe.pipes.reachability.GitRepositoryContext") - @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") - @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) - def test_python_get_symbol_reachability_results( + def _run_reachability_pipeline( self, mock_package_vulnerabilities, mock_collect_symbols, mock_git_context, + file_path, + app_text, + vuln_text, + fixed_text, + expected_results, ): - """ - Test the end-to-end reachability pipeline by analyzing a patch, - computing symbol reachability, and storing the results on the - corresponding codebase resource. - """ - file_path = "app.py" - app_text = (self.data / "python" / file_path).read_text() - vuln_text = (self.data / "python" / "vuln-app.py").read_text() - fixed_text = (self.data / "python" / "fixed-app.py").read_text() - + """Shared helper to run the end-to-end reachability pipeline.""" analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") removed_lines, added_lines = analyzer.compute_changed_lines( @@ -173,6 +166,7 @@ def test_python_get_symbol_reachability_results( self.assertTrue(lang) self.assertTrue(vuln_meta or fixed_meta) + mock_package_vulnerabilities.return_value = [ { "fixed_in_patches": [ @@ -198,11 +192,12 @@ def test_python_get_symbol_reachability_results( } } - resource_file = self.project1.codebase_path / "app.py" + resource_file = self.project1.codebase_path / file_path + resource_file.parent.mkdir(parents=True, exist_ok=True) resource_file.write_text(app_text) collect_and_create_codebase_resources(self.project1) - resource = self.project1.codebaseresources.get(path="app.py") + resource = self.project1.codebaseresources.get(path=file_path) resource.programming_language = lang resource.save() @@ -210,59 +205,83 @@ def test_python_get_symbol_reachability_results( resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") + self.assertEqual(results, expected_results) - self.assertEqual( - results, - [ - { - "symbols_reachability": { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + @patch("scanpipe.pipes.reachability.GitRepositoryContext") + @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") + @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) + def test_python_get_symbol_reachability_results( + self, + mock_package_vulnerabilities, + mock_collect_symbols, + mock_git_context, + ): + """Test the end-to-end reachability pipeline for Python.""" + file_path = "app.py" + app_text = (self.data / "python" / file_path).read_text() + vuln_text = (self.data / "python" / "vuln-app.py").read_text() + fixed_text = (self.data / "python" / "fixed-app.py").read_text() + + expected_results = [ + { + "symbols_reachability": { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "336908735214468b103dbde" + "11c3ffbd2f76ac9212b8514f831cfa078a67892df", + "symbol_name": "debug", + "reachable_from": [], }, - "evidence": [ - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "336908735214468b103dbde" - "11c3ffbd2f76ac9212b8514f831cfa078a67892df", - "symbol_name": "debug", - "reachable_from": [], - }, - { - "called": True, - "defined": True, - "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364" - "e558140239913bfabcc5aa77156460c2eb0a355", - "symbol_name": "serve_report.build_file_path", - "reachable_from": ["serve_report"], - }, - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", - "reachable_from": [], - }, - ], - "fixed_symbols": [ - "debug", - "serve_report", - "serve_report.build_file_path", - ], - "vulnerable_symbols": [ - "debug", - "serve_report", - "serve_report.build_file_path", - ], - "reachability_status": "REACHABLE", - } + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": "762e4f7d03b1bf4359c3ca364" + "e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": ["serve_report"], + }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, + ], + "fixed_symbols": [ + "debug", + "serve_report", + "serve_report.build_file_path", + ], + "vulnerable_symbols": [ + "debug", + "serve_report", + "serve_report.build_file_path", + ], + "reachability_status": "REACHABLE", } - ], + } + ] + + self._run_reachability_pipeline( + mock_package_vulnerabilities, + mock_collect_symbols, + mock_git_context, + file_path, + app_text, + vuln_text, + fixed_text, + expected_results, ) @patch("scanpipe.pipes.reachability.GitRepositoryContext") @@ -274,120 +293,70 @@ def test_java_get_symbol_reachability_results( mock_collect_symbols, mock_git_context, ): - """ - Test the end-to-end reachability pipeline by analyzing a patch, - computing symbol reachability, and storing the results on the - corresponding codebase resource. - """ + """Test the end-to-end reachability pipeline for Java.""" file_path = "app.java" app_text = (self.data / "java" / file_path).read_text() vuln_text = (self.data / "java" / "vuln-app.java").read_text() fixed_text = (self.data / "java" / "fixed-app.java").read_text() - analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") - - removed_lines, added_lines = analyzer.compute_changed_lines( - vulnerable_text=vuln_text, fixed_text=fixed_text - ) - vuln_meta, fixed_meta, lang = analyzer.analyze( - vulnerable_text=vuln_text, - fixed_text=fixed_text, - removed_lines=removed_lines, - added_lines=added_lines, - file_path=file_path, - ) - - self.assertTrue(lang) - self.assertTrue(vuln_meta or fixed_meta) - mock_package_vulnerabilities.return_value = [ + expected_results = [ { - "fixed_in_patches": [ - { + "symbols_reachability": { + "patch": { "vcs_url": "https://github.com/aboutcode-org/test", "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - } - ] + }, + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": None, + "symbol_name": "App", + "reachable_from": [], + }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0" + "a1e6f19e2c4baa421a6cc996fda154", + "symbol_name": "App.serveReport", + "reachable_from": [], + }, + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": None, + "symbol_name": "App.buildFilePath", + "reachable_from": ["App.serveReport"], + }, + ], + "fixed_symbols": [ + "App", + "App.buildFilePath", + "App.serveReport", + ], + "vulnerable_symbols": [ + "App", + "App.buildFilePath", + "App.serveReport", + ], + "reachability_status": "REACHABLE", + } } ] - mock_git_context.return_value.__enter__.return_value.repo = MagicMock() - mock_collect_symbols.return_value = { - lang: { - "vulnerable": { - f"{file_path}::{key}": metadata - for key, metadata in vuln_meta.items() - }, - "fixed": { - f"{file_path}::{key}": metadata - for key, metadata in fixed_meta.items() - }, - } - } - - resource_file = self.project1.codebase_path / "app.java" - resource_file.parent.mkdir(parents=True, exist_ok=True) - resource_file.write_text(app_text) - collect_and_create_codebase_resources(self.project1) - - resource = self.project1.codebaseresources.get(path="app.java") - resource.programming_language = lang - resource.save() - - analyze_and_store_symbol_reachability_results(self.project1) - - resource.refresh_from_db() - results = resource.extra_data.get("symbols_reachability") - - self.assertEqual( - results, - [ - { - "symbols_reachability": { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", - }, - "evidence": [ - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": None, - "symbol_name": "App", - "reachable_from": [], - }, - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0" - "a1e6f19e2c4baa421a6cc996fda154", - "symbol_name": "App.serveReport", - "reachable_from": [], - }, - { - "called": True, - "defined": True, - "imported": False, - "fingerprint": None, - "symbol_name": "App.buildFilePath", - "reachable_from": ["App.serveReport"], - }, - ], - "fixed_symbols": [ - "App", - "App.buildFilePath", - "App.serveReport", - ], - "vulnerable_symbols": [ - "App", - "App.buildFilePath", - "App.serveReport", - ], - "reachability_status": "REACHABLE", - } - } - ], + self._run_reachability_pipeline( + mock_package_vulnerabilities, + mock_collect_symbols, + mock_git_context, + file_path, + app_text, + vuln_text, + fixed_text, + expected_results, ) def test_extract_definitions(self): From 131b68e6660737dae6d2b5740ec287512ce1c753 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 13 Aug 2026 05:46:56 +0300 Subject: [PATCH 30/50] Generate advisory reachability report ( last step in the pipeline ) Split pipeline to multiple steps Build resource_index once per resource Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 225 +++++++++- scanpipe/pipes/reachability.py | 202 ++------- .../tests/pipes/test_symbols_reachability.py | 411 ++++++++++++------ 3 files changed, 542 insertions(+), 296 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index f0939eb179..c77b16b914 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -19,10 +19,21 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json +import os +import shutil +import tempfile +from git import Repo from scanpipe.pipelines import Pipeline -from scanpipe.pipes import reachability +from scanpipe.pipes.reachability import PatchAnalyzer +from scanpipe.pipes.reachability import ReachabilityStatus +from scanpipe.pipes.reachability import ResourceAnalyzer +from scanpipe.pipes.reachability import ResourcePatchMatcher +from scanpipe.pipes.reachability import add_reachability_report +from scanpipe.pipes.reachability import classify_reachability +from scanpipe.pipes.reachability import normalize_text class SymbolReachability(Pipeline): @@ -48,14 +59,212 @@ class SymbolReachability(Pipeline): @classmethod def steps(cls): - return (cls.analyze_symbol_reachability,) + return ( + cls.get_vulnerabilities_patches, + cls.clone_target_repos, + cls.collect_resource_index, + cls.collect_patch_symbols, + cls.collect_and_match_resources, + cls.generate_advisory_reachability_report, + cls.clean_up, + ) + + def get_vulnerabilities_patches(self): + """Get unique patch for all vulnerabilities.""" + patches = {} + for vulnerability in self.project.package_vulnerabilities: + advisory_uid = vulnerability.get("advisory_uid") + for patch in vulnerability.get("fixed_in_patches", []): + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + + p_key = (vcs_url, commit_hash) + if p_key not in patches: + patches[p_key] = { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + "advisory_uids": [], + } + + if advisory_uid and advisory_uid not in patches[p_key]["advisory_uids"]: + patches[p_key]["advisory_uids"].append(advisory_uid) + + self.patches = list(patches.values()) + + def clone_target_repos(self): + """Clone target repos, save the git object""" + self.cloned_git_obj_repos = {} + self.repo_paths = [] + self.patch_symbols = {} + + unique_vcs_urls = {patch["vcs_url"] for patch in self.patches} + for vcs_url in unique_vcs_urls: + try: + repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") + self.repo_paths.append(repo_path) + repo = Repo.clone_from(vcs_url, repo_path) + self.cloned_git_obj_repos[vcs_url] = repo + except Exception as e: + self.log(f"Failed to clone repository {vcs_url}: {e}") + + def collect_resource_index(self): + """Collect resources symbols for each resource exactly once.""" + self.candidate_resources = self.project.codebaseresources.files().filter( + is_binary=False, is_archive=False, is_media=False + ) + self.resource_indexes = {} + for resource in self.candidate_resources: + resource_language = resource.programming_language + + file_content = normalize_text(resource.file_content) + if not file_content: + continue + + resource_analyzer = ResourceAnalyzer( + resource_text=file_content, language=resource_language + ) + + resource_index = resource_analyzer.build_index() + if resource_index: + self.resource_indexes[resource.path] = resource_index + + def collect_patch_symbols(self): + """Collect patch symbols for each patch exactly once.""" + for patch in self.patches: + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + + repo = self.cloned_git_obj_repos.get(vcs_url) + if not repo: + self.patch_symbols[commit_hash] = {} + continue + + try: + patch_analyzer = PatchAnalyzer(repo=repo, commit_hash=commit_hash) + self.patch_symbols[commit_hash] = patch_analyzer.collect_patch_symbols() + except Exception as e: + self.log( + f"Failed to collect patch symbols for {vcs_url}@{commit_hash}: {e}" + ) + self.patch_symbols[commit_hash] = {} + + def collect_and_match_resources(self): + """Match resource symbols against patch symbols.""" + for patch in self.patches: + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + advisory_uids = patch.get("advisory_uids", []) + + patch_symbols_by_language = self.patch_symbols.get(commit_hash, {}) + if not patch_symbols_by_language: + continue + + for resource in self.candidate_resources: + resource_index = self.resource_indexes.get(resource.path) + if not resource_index: + continue - def analyze_symbol_reachability(self): + patch_symbols = patch_symbols_by_language.get( + resource.programming_language + ) + if not patch_symbols: + continue + + vulnerable_symbols = patch_symbols.get("vulnerable", {}) + fixed_symbols = patch_symbols.get("fixed", {}) + + matcher = ResourcePatchMatcher(resource_index=resource_index) + vuln_evidence = matcher.match(vulnerable_symbols) + fixed_evidence = matcher.match(fixed_symbols) + + if not any([vuln_evidence, fixed_evidence]): + continue + + report = { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "advisory_uids": advisory_uids, + "evidence": list(vuln_evidence.values()), + "fixed_symbols": sorted(fixed_evidence.keys()), + "vulnerable_symbols": sorted(vuln_evidence.keys()), + "reachability_status": classify_reachability(vuln_evidence).value, + } + + add_reachability_report( + resource=resource, + commit_hash=commit_hash, + vcs_url=vcs_url, + new_report=report, + ) + + def generate_advisory_reachability_report(self): """ - Perform symbol-level reachability analysis for each patch. This step compares - the AST of patched/vulnerable files against the codebase resources. - Results are stored directly in the 'extra_data' of each CodebaseResource. + Generate reachability report keyed by advisory_uid. + + Each advisory contains its overall reachability status + and the reachability results collected from all resources + and associated patches. + + The overall reachability status is determined using the following + priority order: REACHABLE > UNKNOWN > NOT_REACHABLE + + This means that an advisory is considered reachable if it is reachable + through at least one resource or patch. If no reachable result exists, + but at least one result is unknown, the advisory status is UNKNOWN. + Otherwise, it is NOT_REACHABLE. """ - reachability.analyze_and_store_symbol_reachability_results( - project=self.project, logger=self.log + status_priority = { + ReachabilityStatus.REACHABLE.value: 3, + ReachabilityStatus.UNKNOWN.value: 2, + ReachabilityStatus.NOT_REACHABLE.value: 1, + } + + advisory_reachability_report = {} + for resource in self.candidate_resources: + for report in resource.extra_data.get("symbols_reachability", []): + advisory_uids = report.get("advisory_uids", []) + reachability_status = report.get("reachability_status") + patch = report.get("patch", {}) + + for adv_uid in advisory_uids: + if adv_uid not in advisory_reachability_report: + advisory_reachability_report[adv_uid] = { + "reachable": ReachabilityStatus.NOT_REACHABLE.value, + "details": [], + } + + tool_details = { + "resource_path": resource.path, + "patch": patch, + "reachability_status": reachability_status, + "evidence": report.get("evidence", []), + "vulnerable_symbols": report.get("vulnerable_symbols", []), + "fixed_symbols": report.get("fixed_symbols", []), + } + + advisory_reachability_report[adv_uid]["details"].append( + tool_details + ) + + current_status = advisory_reachability_report[adv_uid]["reachable"] + if status_priority.get( + reachability_status, 0 + ) > status_priority.get(current_status, 0): + advisory_reachability_report[adv_uid]["reachable"] = ( + reachability_status + ) + + reachability_output_path = self.project.get_output_file_path( + "reachability", "json" ) + + with open(reachability_output_path, "w") as f: + json.dump(advisory_reachability_report, f) + + def clean_up(self): + """Clean up cloned repositories""" + for repo_path in self.repo_paths: + if repo_path and os.path.exists(repo_path): + shutil.rmtree(repo_path, ignore_errors=True) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 6bdd8839ab..32d2dc59bc 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -21,7 +21,6 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. # import difflib -import os import shutil import tempfile from enum import Enum @@ -38,9 +37,9 @@ class ReachabilityStatus(str, Enum): - REACHABLE = "REACHABLE" - POTENTIALLY_REACHABLE = "POTENTIALLY_REACHABLE" - NOT_REACHABLE = "NOT_REACHABLE" + REACHABLE = "YES" + UNKNOWN = "UNKNOWN" + NOT_REACHABLE = "NO" def normalize_text(content): @@ -77,33 +76,6 @@ def detect_language_with_scancode(file_path, content): shutil.rmtree(tmp_dir, ignore_errors=True) -class GitRepositoryContext: - def __init__(self, vcs_url): - self.vcs_url = vcs_url - self.repo_path = None - self._repo = None - - def __enter__(self): - self.repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") - try: - self._repo = Repo.clone_from(self.vcs_url, self.repo_path) - return self - except Exception as exc: - self._cleanup() - raise ValueError(f"Failed to clone/checkout {self.vcs_url}") from exc - - def __exit__(self, exc_type, exc_val, exc_tb): - self._cleanup() - - @property - def repo(self) -> Repo: - return self._repo - - def _cleanup(self): - if self.repo_path and os.path.exists(self.repo_path): - shutil.rmtree(self.repo_path, ignore_errors=True) - - class PatchAnalyzer: def __init__(self, repo: Repo, commit_hash): self.repo = repo @@ -276,6 +248,7 @@ def build_symbol_metadata(cls, nodes, extractor): index = extractor.extract_definitions_index() metadata = {} + name_counts = {} for node in nodes: qualified_name = extractor._build_qualified_name(node, index) if not qualified_name: @@ -284,11 +257,8 @@ def build_symbol_metadata(cls, nodes, extractor): body_text = node.text.decode("utf-8", errors="replace") fingerprints = create_sha256_fingerprint(body_text) - key = qualified_name - suffix = 1 - while key in metadata: - suffix += 1 - key = f"{qualified_name}#{suffix}" + count = name_counts[qualified_name] = name_counts.get(qualified_name, 0) + 1 + key = qualified_name if count == 1 else f"{qualified_name}#{count}" metadata[key] = { "qualified_name": qualified_name, @@ -378,134 +348,6 @@ def analyze( return vuln_meta, fixed_meta, language -def generate_reachability_report(patch, repo, candidate_resources, logger=None): - """ - Generate and store a symbol reachability report for a single - vulnerability patch against a set of candidate codebase resources. - - - Creates a PatchAnalyzer for the given commit. - - Collects changed symbols from the patch, grouped by language. - - For each candidate resource whose language matches a patch - language, builds a ResourceAnalyzer index and uses - ResourcePatchMatcher to match vulnerable and fixed - symbols. - - If matches are found, constructs a report dict containing - evidence, matched symbol names, and a ReachabilityStatus. - - Appends the report to the resource's extra_data under the - symbols_reachability key (avoiding duplicates for the same - commit hash). - - """ - vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - - try: - patch_analyzer = PatchAnalyzer(repo=repo, commit_hash=commit_hash) - patch_symbols_by_language = patch_analyzer.collect_patch_symbols() - - if not patch_symbols_by_language: - return - - for resource in candidate_resources: - resource_language = resource.programming_language - patch_symbols = patch_symbols_by_language.get(resource_language) - - if not patch_symbols: - continue - - file_content = normalize_text(resource.file_content) - if not file_content: - continue - - resource_analyzer = ResourceAnalyzer( - resource_text=file_content, language=resource_language - ) - resource_index = resource_analyzer.build_index() - - if not resource_index: - continue - - matcher = ResourcePatchMatcher(resource_index=resource_index) - vuln_evidence = matcher.match(patch_symbols["vulnerable"]) - fixed_evidence = matcher.match(patch_symbols["fixed"]) - - if not any([vuln_evidence, fixed_evidence]): - continue - - report = { - "symbols_reachability": { - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, - "evidence": list(vuln_evidence.values()), - "fixed_symbols": sorted(fixed_evidence.keys()), - "vulnerable_symbols": sorted(vuln_evidence.keys()), - "reachability_status": classify_reachability(vuln_evidence).value, - } - } - - existing = resource.extra_data.get("symbols_reachability") - reports = existing if isinstance(existing, list) else [] - - if not any( - r.get("patch", {}).get("commit_hash") == commit_hash for r in reports - ): - reports.append(report) - resource.update_extra_data({"symbols_reachability": reports}) - - except Exception as e: - if logger: - logger( - f"Failed to collect symbol reachability for " - f"{vcs_url}@{commit_hash}: {e}" - ) - - -def analyze_and_store_symbol_reachability_results(project, logger=None): - """ - Analyze all vulnerability patches for a project and store the - resulting symbol reachability reports - - The function iterates over all package vulnerabilities associated - with the project, groups their patches by repository URL, clones - each repository (using :class:`GitRepositoryContext`), checks out - each patch's commit, and calls :func:`generate_reachability_report` - to determine whether the vulnerable/fixed symbols from the patch - are reachable in the project's codebase resources. - - Only non-binary, non-archive, non-media files are considered as - candidate resources. - """ - candidate_resources = project.codebaseresources.files().filter( - is_binary=False, is_archive=False, is_media=False - ) - - for vulnerability in project.package_vulnerabilities: - patches_by_repo = {} - for patch in vulnerability.get("fixed_in_patches", []): - vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - if vcs_url and commit_hash: - patches_by_repo.setdefault(vcs_url, []).append(patch) - - for vcs_url, patches in patches_by_repo.items(): - try: - with GitRepositoryContext(vcs_url) as git_context: - for patch in patches: - commit_hash = patch.get("commit_hash") - git_context.repo.git.checkout(commit_hash) - generate_reachability_report( - patch, git_context.repo, candidate_resources, logger - ) - except Exception as e: - if logger: - logger( - f"Failed to process repository {vcs_url} " - f"for symbol reachability: {e}" - ) - - def classify_reachability(evidence): """ Classify the reachability status of a vulnerability based on the @@ -526,7 +368,7 @@ def classify_reachability(evidence): return ReachabilityStatus.REACHABLE if (is_imported or is_defined) and not is_exact: - status = ReachabilityStatus.POTENTIALLY_REACHABLE + status = ReachabilityStatus.UNKNOWN return status @@ -685,3 +527,33 @@ def match(self, patch_symbols_metadata): entry["fingerprint"] = fingerprint return matched + + +def add_reachability_report(resource, commit_hash, vcs_url, new_report): + """ + Add a reachability report for commit_hash and vcs_url. + If duplicates exist, replace the old report with the new one. + """ + cleaned_reports = [] + replaced = False + + old_reports = resource.extra_data.get("symbols_reachability", []) + for old_report in old_reports: + actual_report = old_report.get("symbols_reachability", old_report) + + patch_info = actual_report.get("patch", {}) + old_commit_hash = patch_info.get("commit_hash") + old_vcs_url = patch_info.get("vcs_url") + + if old_commit_hash == commit_hash and old_vcs_url == vcs_url: + if not replaced: + cleaned_reports.append(new_report) + replaced = True + # Skip old duplicate + else: + cleaned_reports.append(actual_report) + + if not replaced: + cleaned_reports.append(new_report) + + resource.update_extra_data({"symbols_reachability": cleaned_reports}) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 4cc0ad74cf..7094729cb9 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -1,6 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # -# http://nexb.com and https://github.com/nexB/scancode.io +# http://nexb.com and https://github.com/aboutcode-org/scancode.io # The ScanCode.io software is licensed under the Apache License version 2.0. # Data generated with ScanCode.io is provided as-is without warranties. # ScanCode is a trademark of nexB Inc. @@ -18,7 +18,10 @@ # for any legal advice. # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. -# Visit https://github.com/nexB/scancode.io for support and download. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import json +import os import shutil import sys import tempfile @@ -34,7 +37,6 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus -from scanpipe.pipes.reachability import analyze_and_store_symbol_reachability_results from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor @@ -106,44 +108,208 @@ def test_end_to_end_symbol_reachability_pipeline( } ] - analyze_and_store_symbol_reachability_results(self.project1) + run = self.project1.add_pipeline("analyze_symbols_reachability") + pipeline = run.make_pipeline_instance() + pipeline.execute() + resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") expected_results = [ { - "symbols_reachability": { + "patch": { + "vcs_url": repo_url, + "commit_hash": fixed_commit.hexsha, + }, + "advisory_uids": [], + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "e66988810af452f77d43efd302fea4c" + "1d2f246d282c4ea8982b0152c2231ac17", + "symbol_name": "process_data", + "reachable_from": [], + } + ], + "fixed_symbols": ["process_data"], + "vulnerable_symbols": ["process_data"], + "reachability_status": "YES", + } + ] + + self.assertEqual(results, expected_results) + finally: + shutil.rmtree(vcs_dir, ignore_errors=True) + + def test_generate_advisory_reachability_report(self): + """Test the generation of the advisory reachability report""" + run = self.project1.add_pipeline("analyze_symbols_reachability") + pipeline = run.make_pipeline_instance() + + with tempfile.TemporaryDirectory() as tmpdir: + output_file = os.path.join(tmpdir, "reachability.json") + pipeline.project.get_output_file_path = MagicMock(return_value=output_file) + + res1 = MagicMock() + res1.path = "src/file1.py" + res1.extra_data = { + "symbols_reachability": [ + { + "advisory_uids": ["AVID-1", "AVID-2"], + "reachability_status": ReachabilityStatus.REACHABLE.value, "patch": { - "vcs_url": repo_url, - "commit_hash": fixed_commit.hexsha, + "vcs_url": "https://example.com", + "commit_hash": "abc123", }, "evidence": [ { - "called": False, - "defined": True, + "symbol_name": "vuln_sym1", + "called": True, + "defined": False, "imported": False, - "fingerprint": "e66988810af452f77d43efd302fea4c" - "1d2f246d282c4ea8982b0152c2231ac17", - "symbol_name": "process_data", + "fingerprint": "88ad9e67c53aa5f7c4" + "3ec4aa52ed34b7930068c9", "reachable_from": [], } ], - "fixed_symbols": ["process_data"], - "vulnerable_symbols": ["process_data"], - "reachability_status": "REACHABLE", + "vulnerable_symbols": ["vuln_sym1"], + "fixed_symbols": ["fixed_sym1"], } - } - ] + ] + } - self.assertEqual(results, expected_results) - finally: - shutil.rmtree(vcs_dir, ignore_errors=True) + res2 = MagicMock() + res2.path = "src/file2.py" + res2.extra_data = { + "symbols_reachability": [ + { + "advisory_uids": ["AVID-1"], + "reachability_status": ReachabilityStatus.UNKNOWN.value, + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "def456", + }, + "evidence": [], + "vulnerable_symbols": [], + "fixed_symbols": [], + } + ] + } + + res3 = MagicMock() + res3.path = "src/file3.py" + res3.extra_data = { + "symbols_reachability": [ + { + "advisory_uids": ["AVID-2"], + "reachability_status": ReachabilityStatus.NOT_REACHABLE.value, + "patch": { + "vcs_url": "https://example2.com", + "commit_hash": "46a4asf", + }, + "evidence": [], + "vulnerable_symbols": [], + "fixed_symbols": [], + } + ] + } + + res_empty = MagicMock() + res_empty.path = "src/empty.py" + res_empty.extra_data = {} + + pipeline.candidate_resources = [res2, res3, res_empty, res1] + pipeline.generate_advisory_reachability_report() + + with open(output_file) as f: + report = json.load(f) + + expected_report = { + "AVID-1": { + "reachable": "YES", + "details": [ + { + "resource_path": "src/file2.py", + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "def456", + }, + "reachability_status": "UNKNOWN", + "evidence": [], + "vulnerable_symbols": [], + "fixed_symbols": [], + }, + { + "resource_path": "src/file1.py", + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "abc123", + }, + "reachability_status": "YES", + "evidence": [ + { + "symbol_name": "vuln_sym1", + "called": True, + "defined": False, + "imported": False, + "fingerprint": "88ad9e67c53aa5f7c43e" + "c4aa52ed34b7930068c9", + "reachable_from": [], + } + ], + "vulnerable_symbols": ["vuln_sym1"], + "fixed_symbols": ["fixed_sym1"], + }, + ], + }, + "AVID-2": { + "reachable": "YES", + "details": [ + { + "resource_path": "src/file3.py", + "patch": { + "vcs_url": "https://example2.com", + "commit_hash": "46a4asf", + }, + "reachability_status": "NO", + "evidence": [], + "vulnerable_symbols": [], + "fixed_symbols": [], + }, + { + "resource_path": "src/file1.py", + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "abc123", + }, + "reachability_status": "YES", + "evidence": [ + { + "symbol_name": "vuln_sym1", + "called": True, + "defined": False, + "imported": False, + "fingerprint": "88ad9e67c53aa5f7c4" + "3ec4aa52ed34b7930068c9", + "reachable_from": [], + } + ], + "vulnerable_symbols": ["vuln_sym1"], + "fixed_symbols": ["fixed_sym1"], + }, + ], + }, + } + + self.assertEqual(report, expected_report) def _run_reachability_pipeline( self, mock_package_vulnerabilities, mock_collect_symbols, - mock_git_context, + mock_repo, file_path, app_text, vuln_text, @@ -178,7 +344,6 @@ def _run_reachability_pipeline( } ] - mock_git_context.return_value.__enter__.return_value.repo = MagicMock() mock_collect_symbols.return_value = { lang: { "vulnerable": { @@ -201,20 +366,22 @@ def _run_reachability_pipeline( resource.programming_language = lang resource.save() - analyze_and_store_symbol_reachability_results(self.project1) + run = self.project1.add_pipeline("analyze_symbols_reachability") + pipeline = run.make_pipeline_instance() + pipeline.execute() resource.refresh_from_db() results = resource.extra_data.get("symbols_reachability") self.assertEqual(results, expected_results) - @patch("scanpipe.pipes.reachability.GitRepositoryContext") + @patch("scanpipe.pipelines.analyze_symbols_reachability.Repo") @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_python_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, - mock_git_context, + mock_repo, ): """Test the end-to-end reachability pipeline for Python.""" file_path = "app.py" @@ -224,59 +391,58 @@ def test_python_get_symbol_reachability_results( expected_results = [ { - "symbols_reachability": { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "advisory_uids": [], + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "336908735214468b103dbde" + "11c3ffbd2f76ac9212b8514f831cfa078a67892df", + "symbol_name": "debug", + "reachable_from": [], }, - "evidence": [ - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "336908735214468b103dbde" - "11c3ffbd2f76ac9212b8514f831cfa078a67892df", - "symbol_name": "debug", - "reachable_from": [], - }, - { - "called": True, - "defined": True, - "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364" - "e558140239913bfabcc5aa77156460c2eb0a355", - "symbol_name": "serve_report.build_file_path", - "reachable_from": ["serve_report"], - }, - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", - "symbol_name": "serve_report", - "reachable_from": [], - }, - ], - "fixed_symbols": [ - "debug", - "serve_report", - "serve_report.build_file_path", - ], - "vulnerable_symbols": [ - "debug", - "serve_report", - "serve_report.build_file_path", - ], - "reachability_status": "REACHABLE", - } + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": "762e4f7d03b1bf4359c3ca364" + "e558140239913bfabcc5aa77156460c2eb0a355", + "symbol_name": "serve_report.build_file_path", + "reachable_from": ["serve_report"], + }, + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "d7675efb263896da2a3c0067951183" + "3553907e7e6ea619115a6dfc8625c3457e", + "symbol_name": "serve_report", + "reachable_from": [], + }, + ], + "fixed_symbols": [ + "debug", + "serve_report", + "serve_report.build_file_path", + ], + "vulnerable_symbols": [ + "debug", + "serve_report", + "serve_report.build_file_path", + ], + "reachability_status": "YES", } ] self._run_reachability_pipeline( mock_package_vulnerabilities, mock_collect_symbols, - mock_git_context, + mock_repo, file_path, app_text, vuln_text, @@ -284,14 +450,14 @@ def test_python_get_symbol_reachability_results( expected_results, ) - @patch("scanpipe.pipes.reachability.GitRepositoryContext") + @patch("scanpipe.pipelines.analyze_symbols_reachability.Repo") @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_java_get_symbol_reachability_results( self, mock_package_vulnerabilities, mock_collect_symbols, - mock_git_context, + mock_repo, ): """Test the end-to-end reachability pipeline for Java.""" file_path = "app.java" @@ -301,57 +467,56 @@ def test_java_get_symbol_reachability_results( expected_results = [ { - "symbols_reachability": { - "patch": { - "vcs_url": "https://github.com/aboutcode-org/test", - "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "advisory_uids": [], + "evidence": [ + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": None, + "symbol_name": "App", + "reachable_from": [], }, - "evidence": [ - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": None, - "symbol_name": "App", - "reachable_from": [], - }, - { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0" - "a1e6f19e2c4baa421a6cc996fda154", - "symbol_name": "App.serveReport", - "reachable_from": [], - }, - { - "called": True, - "defined": True, - "imported": False, - "fingerprint": None, - "symbol_name": "App.buildFilePath", - "reachable_from": ["App.serveReport"], - }, - ], - "fixed_symbols": [ - "App", - "App.buildFilePath", - "App.serveReport", - ], - "vulnerable_symbols": [ - "App", - "App.buildFilePath", - "App.serveReport", - ], - "reachability_status": "REACHABLE", - } + { + "called": False, + "defined": True, + "imported": False, + "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0" + "a1e6f19e2c4baa421a6cc996fda154", + "symbol_name": "App.serveReport", + "reachable_from": [], + }, + { + "called": True, + "defined": True, + "imported": False, + "fingerprint": None, + "symbol_name": "App.buildFilePath", + "reachable_from": ["App.serveReport"], + }, + ], + "fixed_symbols": [ + "App", + "App.buildFilePath", + "App.serveReport", + ], + "vulnerable_symbols": [ + "App", + "App.buildFilePath", + "App.serveReport", + ], + "reachability_status": "YES", } ] self._run_reachability_pipeline( mock_package_vulnerabilities, mock_collect_symbols, - mock_git_context, + mock_repo, file_path, app_text, vuln_text, @@ -467,23 +632,23 @@ def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) self.assertEqual( - classify_reachability({"evidence": {}}), ReachabilityStatus.NOT_REACHABLE + classify_reachability({"symbol1": {}}), ReachabilityStatus.NOT_REACHABLE ) self.assertEqual( - classify_reachability({"evidence": {"fingerprint": "hash123"}}), + classify_reachability({"symbol1": {"fingerprint": "hash123"}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"evidence": {"imported": True, "called": True}}), + classify_reachability({"symbol1": {"imported": True, "called": True}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"evidence": {"imported": True, "called": False}}), - ReachabilityStatus.POTENTIALLY_REACHABLE, + classify_reachability({"symbol1": {"imported": True, "called": False}}), + ReachabilityStatus.UNKNOWN, ) self.assertEqual( - classify_reachability({"evidence": {"imported": False, "called": False}}), + classify_reachability({"symbol1": {"imported": False, "called": False}}), ReachabilityStatus.NOT_REACHABLE, ) From 69fc18a7556ae295f936af81be62b1cf4d067aeb Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 14 Aug 2026 06:07:50 +0300 Subject: [PATCH 31/50] Fix Formating and typo in test Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 1 + scanpipe/pipes/reachability.py | 2 +- .../tests/pipes/test_symbols_reachability.py | 22 +++++++++---------- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index c77b16b914..6b93c3a94f 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -19,6 +19,7 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. + import json import os import shutil diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 32d2dc59bc..c0ea1addc0 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -19,7 +19,7 @@ # # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -# + import difflib import shutil import tempfile diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 7094729cb9..1a76336c7a 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -135,7 +135,7 @@ def test_end_to_end_symbol_reachability_pipeline( ], "fixed_symbols": ["process_data"], "vulnerable_symbols": ["process_data"], - "reachability_status": "YES", + "reachability_status": ReachabilityStatus.REACHABLE.value, } ] @@ -228,7 +228,7 @@ def test_generate_advisory_reachability_report(self): expected_report = { "AVID-1": { - "reachable": "YES", + "reachable": ReachabilityStatus.REACHABLE.value, "details": [ { "resource_path": "src/file2.py", @@ -247,7 +247,7 @@ def test_generate_advisory_reachability_report(self): "vcs_url": "https://example.com", "commit_hash": "abc123", }, - "reachability_status": "YES", + "reachability_status": ReachabilityStatus.REACHABLE.value, "evidence": [ { "symbol_name": "vuln_sym1", @@ -265,7 +265,7 @@ def test_generate_advisory_reachability_report(self): ], }, "AVID-2": { - "reachable": "YES", + "reachable": ReachabilityStatus.REACHABLE.value, "details": [ { "resource_path": "src/file3.py", @@ -284,7 +284,7 @@ def test_generate_advisory_reachability_report(self): "vcs_url": "https://example.com", "commit_hash": "abc123", }, - "reachability_status": "YES", + "reachability_status": ReachabilityStatus.REACHABLE.value, "evidence": [ { "symbol_name": "vuln_sym1", @@ -435,7 +435,7 @@ def test_python_get_symbol_reachability_results( "serve_report", "serve_report.build_file_path", ], - "reachability_status": "YES", + "reachability_status": ReachabilityStatus.REACHABLE.value, } ] @@ -509,7 +509,7 @@ def test_java_get_symbol_reachability_results( "App.buildFilePath", "App.serveReport", ], - "reachability_status": "YES", + "reachability_status": ReachabilityStatus.REACHABLE.value, } ] @@ -632,19 +632,19 @@ def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) self.assertEqual( - classify_reachability({"symbol1": {}}), ReachabilityStatus.NOT_REACHABLE + classify_reachability({"evidence": {}}), ReachabilityStatus.NOT_REACHABLE ) self.assertEqual( - classify_reachability({"symbol1": {"fingerprint": "hash123"}}), + classify_reachability({"evidence": {"fingerprint": "hash123"}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"symbol1": {"imported": True, "called": True}}), + classify_reachability({"evidence": {"imported": True, "called": True}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"symbol1": {"imported": True, "called": False}}), + classify_reachability({"evidence": {"imported": True, "called": False}}), ReachabilityStatus.UNKNOWN, ) self.assertEqual( From 98350c33d52d68f136c7e78de8aa48d385531007 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 14 Aug 2026 06:11:00 +0300 Subject: [PATCH 32/50] Remove type hints Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index c0ea1addc0..49fa6964cb 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -26,7 +26,6 @@ from enum import Enum from pathlib import Path -from git import Repo from git.diff import NULL_TREE from scancode.api import get_file_info @@ -77,7 +76,7 @@ def detect_language_with_scancode(file_path, content): class PatchAnalyzer: - def __init__(self, repo: Repo, commit_hash): + def __init__(self, repo, commit_hash): self.repo = repo self.commit = repo.commit(commit_hash) self.parent_commit = self.commit.parents[0] if self.commit.parents else None From 9ce4cb4bc3b05024a4b9feeba4274e7ad7a61491 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 14 Aug 2026 06:15:16 +0300 Subject: [PATCH 33/50] Remove type hints for symbols.py Signed-off-by: ziad hany --- scanpipe/pipes/symbols.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index f86630b4f5..a4da7e1978 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -179,7 +179,7 @@ def _collect_and_store_tree_sitter_symbols_and_strings(resource): @cache -def load_language(language: str): +def load_language(language): from source_inspector import symbols_tree_sitter if language not in symbols_tree_sitter.TS_LANGUAGE_WHEELS: @@ -392,7 +392,7 @@ def __init__(self, lang_query: LanguageQuery, root_node): self.root_node = root_node self.syntax_config = lang_query.syntax_config - def _build_qualified_name(self, node, index: dict) -> str: + def _build_qualified_name(self, node, index): """ Build fully qualified names internally (e.g., ClassName.function_name or ClassName::function_name). @@ -410,7 +410,7 @@ def _build_qualified_name(self, node, index: dict) -> str: def extract_definitions_index(self): """Build the index of definitions with fully qualified names.""" - index: dict[int, dict] = {} + index = {} for node, name in self.lang_query.get_functions(self.root_node): index[node.id] = {"node": node, "name": name, "kind": "functions"} @@ -428,7 +428,7 @@ def extract_definitions_index(self): return index - def extract_changed_symbols(self, changed_lines: list[int]): + def extract_changed_symbols(self, changed_lines): """Map changed line numbers to their enclosing symbol nodes.""" if self.root_node is None or not changed_lines: return [] @@ -474,8 +474,8 @@ def extract_imports(self): separator = self.syntax_config.get("separator", ".") wildcard_sym = self.syntax_config.get("wildcard_symbol") - import_map: dict[str, str | list[str]] = {} - wildcard_modules: list[str] = [] + import_map = {} + wildcard_modules = [] for module_name, pairs in self.lang_query.get_imports(self.root_node): for imp_name, alias in pairs: From c89c64eaecbf06a7abaf06b789676f6f16168eee Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 14 Aug 2026 14:15:39 +0300 Subject: [PATCH 34/50] Remove type hints for symbols.py Signed-off-by: ziad hany --- scanpipe/pipes/symbols.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index a4da7e1978..8abd989bdb 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -228,7 +228,7 @@ def __init__(self): Query(self.ts_language, source) if source else None ) - def parse_code_to_ast(self, code_text: str): + def parse_code_to_ast(self, code_text): from source_inspector import symbols_tree_sitter from tree_sitter import Parser @@ -239,7 +239,7 @@ def parse_code_to_ast(self, code_text: str): code_text.encode("utf-8") ), symbols_tree_sitter.TS_LANGUAGE_WHEELS[self.language_name] - def run_query(self, kind: str, root_node): + def run_query(self, kind, root_node): query = self._compiled_queries.get(kind) return query.matches(root_node) if query else [] From 2f14b410e44312b8bff99200a218a42ff15f96bf Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 14 Aug 2026 18:07:10 +0300 Subject: [PATCH 35/50] Fix a typo in docs Simplify add_reachability_report function Remove type hints for symbols.py Signed-off-by: ziad hany --- scanpipe/pipelines/analyze_symbols_reachability.py | 5 ++++- scanpipe/pipes/reachability.py | 6 ++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 6b93c3a94f..d7bbb532ed 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -49,9 +49,12 @@ class SymbolReachability(Pipeline): The analysis checks if vulnerable symbols are defined, imported, called, or exactly match a fingerprint within the project files. The results, including - evidence and a reachability status (REACHABLE, POTENTIALLY_REACHABLE, or + evidence and a reachability status (REACHABLE, UNKNOWN, or NOT_REACHABLE), are stored in the `extra_data` of the matching resources under the `symbols_reachability` key. + + Finally, a summary report is generated for each vulnerability + advisory and saved as a JSON output file. """ download_inputs = False diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 49fa6964cb..65be737614 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -538,9 +538,7 @@ def add_reachability_report(resource, commit_hash, vcs_url, new_report): old_reports = resource.extra_data.get("symbols_reachability", []) for old_report in old_reports: - actual_report = old_report.get("symbols_reachability", old_report) - - patch_info = actual_report.get("patch", {}) + patch_info = old_report.get("patch", {}) old_commit_hash = patch_info.get("commit_hash") old_vcs_url = patch_info.get("vcs_url") @@ -550,7 +548,7 @@ def add_reachability_report(resource, commit_hash, vcs_url, new_report): replaced = True # Skip old duplicate else: - cleaned_reports.append(actual_report) + cleaned_reports.append(old_report) if not replaced: cleaned_reports.append(new_report) From 9a6d42dc46c35e3f7b7cc729ccaf8cc067d1184e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Sat, 15 Aug 2026 00:02:34 +0300 Subject: [PATCH 36/50] Fix a typo in error message Signed-off-by: ziad hany --- scanpipe/pipelines/analyze_symbols_reachability.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index d7bbb532ed..16b5941896 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -109,7 +109,7 @@ def clone_target_repos(self): repo = Repo.clone_from(vcs_url, repo_path) self.cloned_git_obj_repos[vcs_url] = repo except Exception as e: - self.log(f"Failed to clone repository {vcs_url}: {e}") + self.log(f"Failed to clone repository {vcs_url}: {e!r}") def collect_resource_index(self): """Collect resources symbols for each resource exactly once.""" @@ -148,7 +148,8 @@ def collect_patch_symbols(self): self.patch_symbols[commit_hash] = patch_analyzer.collect_patch_symbols() except Exception as e: self.log( - f"Failed to collect patch symbols for {vcs_url}@{commit_hash}: {e}" + f"Failed to collect patch " + f"symbols for {vcs_url}@{commit_hash}: {e!r}" ) self.patch_symbols[commit_hash] = {} From 01f294cfd980b114c982fb6b97d146b44e63d047 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 18 Aug 2026 17:10:04 +0300 Subject: [PATCH 37/50] Update the pipeline to clone repo once and collect_patch_symbols for all commits Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 76 ++++++++----------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 16b5941896..89df08b512 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -21,8 +21,6 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import json -import os -import shutil import tempfile from git import Repo @@ -65,12 +63,10 @@ class SymbolReachability(Pipeline): def steps(cls): return ( cls.get_vulnerabilities_patches, - cls.clone_target_repos, cls.collect_resource_index, cls.collect_patch_symbols, cls.collect_and_match_resources, cls.generate_advisory_reachability_report, - cls.clean_up, ) def get_vulnerabilities_patches(self): @@ -95,22 +91,6 @@ def get_vulnerabilities_patches(self): self.patches = list(patches.values()) - def clone_target_repos(self): - """Clone target repos, save the git object""" - self.cloned_git_obj_repos = {} - self.repo_paths = [] - self.patch_symbols = {} - - unique_vcs_urls = {patch["vcs_url"] for patch in self.patches} - for vcs_url in unique_vcs_urls: - try: - repo_path = tempfile.mkdtemp(prefix="symbol-reachability-") - self.repo_paths.append(repo_path) - repo = Repo.clone_from(vcs_url, repo_path) - self.cloned_git_obj_repos[vcs_url] = repo - except Exception as e: - self.log(f"Failed to clone repository {vcs_url}: {e!r}") - def collect_resource_index(self): """Collect resources symbols for each resource exactly once.""" self.candidate_resources = self.project.codebaseresources.files().filter( @@ -133,25 +113,39 @@ def collect_resource_index(self): self.resource_indexes[resource.path] = resource_index def collect_patch_symbols(self): - """Collect patch symbols for each patch exactly once.""" + """ + For each unique repo: clone it once, collect patch symbols for all + related commits, then remove the local clone before moving on + """ + self.patch_symbols = {} + patches_by_repo = {} for patch in self.patches: vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - - repo = self.cloned_git_obj_repos.get(vcs_url) - if not repo: - self.patch_symbols[commit_hash] = {} - continue - - try: - patch_analyzer = PatchAnalyzer(repo=repo, commit_hash=commit_hash) - self.patch_symbols[commit_hash] = patch_analyzer.collect_patch_symbols() - except Exception as e: - self.log( - f"Failed to collect patch " - f"symbols for {vcs_url}@{commit_hash}: {e!r}" - ) - self.patch_symbols[commit_hash] = {} + patches_by_repo.setdefault(vcs_url, []).append(patch) + + for vcs_url, repo_patches in patches_by_repo.items(): + with tempfile.TemporaryDirectory( + prefix="symbol-reachability-" + ) as repo_path: + try: + repo = Repo.clone_from(vcs_url, repo_path) + except Exception as e: + raise Exception(f"Failed to clone repository {vcs_url}: {e!r}") + + try: + for patch in repo_patches: + commit_hash = patch.get("commit_hash") + patch_analyzer = PatchAnalyzer( + repo=repo, commit_hash=commit_hash + ) + self.patch_symbols[commit_hash] = ( + patch_analyzer.collect_patch_symbols() + ) + except Exception as e: + raise Exception( + f"Failed to collect patch symbols " + f"for {vcs_url}, patch: {repo_patches}: {e!r}" + ) def collect_and_match_resources(self): """Match resource symbols against patch symbols.""" @@ -266,10 +260,4 @@ def generate_advisory_reachability_report(self): ) with open(reachability_output_path, "w") as f: - json.dump(advisory_reachability_report, f) - - def clean_up(self): - """Clean up cloned repositories""" - for repo_path in self.repo_paths: - if repo_path and os.path.exists(repo_path): - shutil.rmtree(repo_path, ignore_errors=True) + json.dump(advisory_reachability_report, f, indent=2) From 40f5f0ef92bb07715aaf96dff1c057e325dcff8d Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 20 Aug 2026 03:59:08 +0300 Subject: [PATCH 38/50] Try to fix bug in vulnerability dependency Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 5 +++- scanpipe/pipes/reachability.py | 11 +------ .../tests/data/reachability/python/app.py | 4 +++ .../data/reachability/python/fixed-app.py | 4 +++ .../data/reachability/python/vuln-app.py | 4 +++ .../tests/pipes/test_symbols_reachability.py | 29 +++++++++---------- 6 files changed, 31 insertions(+), 26 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 89df08b512..8a9aa87a36 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -72,7 +72,10 @@ def steps(cls): def get_vulnerabilities_patches(self): """Get unique patch for all vulnerabilities.""" patches = {} - for vulnerability in self.project.package_vulnerabilities: + for vulnerability in ( + self.project.package_vulnerabilities + + self.project.dependency_vulnerabilities + ): advisory_uid = vulnerability.get("advisory_uid") for patch in vulnerability.get("fixed_in_patches", []): vcs_url = patch.get("vcs_url") diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 65be737614..b4960d15bb 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -479,13 +479,6 @@ def match(self, patch_symbols_metadata): for metadata in patch_symbols_metadata.values(): qualified_name = metadata["qualified_name"] fingerprint = metadata["fingerprint"] - - short_name = ( - qualified_name.rsplit(self.separator, 1)[-1] - if self.separator in qualified_name - else qualified_name - ) - defined = qualified_name in self.definitions fingerprint_hit = bool(fingerprint and fingerprint in self.fingerprints) @@ -494,9 +487,7 @@ def match(self, patch_symbols_metadata): or qualified_name in self.imports.values() ) - callers = set() - callers.update(self.callers_of.get(short_name, set())) - callers.update(self.callers_of.get(qualified_name, set())) + callers = set(self.callers_of.get(qualified_name, set())) called = bool(callers) if not (defined or fingerprint_hit or called or imported): diff --git a/scanpipe/tests/data/reachability/python/app.py b/scanpipe/tests/data/reachability/python/app.py index 32eebc2fa1..dae9bce750 100644 --- a/scanpipe/tests/data/reachability/python/app.py +++ b/scanpipe/tests/data/reachability/python/app.py @@ -32,6 +32,10 @@ def build_file_path(filename): return "Error: File not found" +def handle_request(req): + return serve_report(req) + + def unrelated_top_level_function(): """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/python/fixed-app.py b/scanpipe/tests/data/reachability/python/fixed-app.py index 30470b6cda..23f5633e80 100644 --- a/scanpipe/tests/data/reachability/python/fixed-app.py +++ b/scanpipe/tests/data/reachability/python/fixed-app.py @@ -38,6 +38,10 @@ def build_file_path(filename): return "Error: File not found" +def handle_request(req): + return serve_report(req) + + def unrelated_top_level_function(): """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/data/reachability/python/vuln-app.py b/scanpipe/tests/data/reachability/python/vuln-app.py index 32eebc2fa1..dae9bce750 100644 --- a/scanpipe/tests/data/reachability/python/vuln-app.py +++ b/scanpipe/tests/data/reachability/python/vuln-app.py @@ -32,6 +32,10 @@ def build_file_path(filename): return "Error: File not found" +def handle_request(req): + return serve_report(req) + + def unrelated_top_level_function(): """Test AST node boundaries.""" return "I am just here to add AST complexity." diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 1a76336c7a..88f51c9b5b 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -407,22 +407,22 @@ def test_python_get_symbol_reachability_results( "reachable_from": [], }, { - "called": True, + "called": False, "defined": True, "imported": False, "fingerprint": "762e4f7d03b1bf4359c3ca364" "e558140239913bfabcc5aa77156460c2eb0a355", "symbol_name": "serve_report.build_file_path", - "reachable_from": ["serve_report"], + "reachable_from": [], }, { - "called": False, + "called": True, "defined": True, "imported": False, "fingerprint": "d7675efb263896da2a3c0067951183" "3553907e7e6ea619115a6dfc8625c3457e", "symbol_name": "serve_report", - "reachable_from": [], + "reachable_from": ["handle_request"], }, ], "fixed_symbols": [ @@ -474,40 +474,39 @@ def test_java_get_symbol_reachability_results( "advisory_uids": [], "evidence": [ { + "symbol_name": "App", "called": False, "defined": True, "imported": False, "fingerprint": None, - "symbol_name": "App", "reachable_from": [], }, { + "symbol_name": "App.serveReport", "called": False, "defined": True, "imported": False, - "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0" - "a1e6f19e2c4baa421a6cc996fda154", - "symbol_name": "App.serveReport", - "reachable_from": [], + "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0a1e6f19e2c4baa421a6cc996fda154", + "reachable_from": [] }, { - "called": True, + "symbol_name": "App.buildFilePath", + "called": False, "defined": True, "imported": False, "fingerprint": None, - "symbol_name": "App.buildFilePath", - "reachable_from": ["App.serveReport"], - }, + "reachable_from": [] + } ], "fixed_symbols": [ "App", "App.buildFilePath", - "App.serveReport", + "App.serveReport" ], "vulnerable_symbols": [ "App", "App.buildFilePath", - "App.serveReport", + "App.serveReport" ], "reachability_status": ReachabilityStatus.REACHABLE.value, } From a36ba7e50d7b15280af76f549095f8cc8da10aaf Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 24 Aug 2026 04:45:57 +0300 Subject: [PATCH 39/50] Update the pipeline to have a test for resource_patch_matcher Refactor the pipeline Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 54 +- scanpipe/pipes/reachability.py | 140 ++++-- scanpipe/pipes/symbols.py | 42 +- .../tests/pipes/test_symbols_reachability.py | 471 +++++++++++++----- 4 files changed, 502 insertions(+), 205 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 8a9aa87a36..186ea7345e 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -47,7 +47,7 @@ class SymbolReachability(Pipeline): The analysis checks if vulnerable symbols are defined, imported, called, or exactly match a fingerprint within the project files. The results, including - evidence and a reachability status (REACHABLE, UNKNOWN, or + tool_details and a reachability status (REACHABLE, UNKNOWN, or NOT_REACHABLE), are stored in the `extra_data` of the matching resources under the `symbols_reachability` key. @@ -176,10 +176,10 @@ def collect_and_match_resources(self): fixed_symbols = patch_symbols.get("fixed", {}) matcher = ResourcePatchMatcher(resource_index=resource_index) - vuln_evidence = matcher.match(vulnerable_symbols) - fixed_evidence = matcher.match(fixed_symbols) + vuln_details = matcher.match(vulnerable_symbols) + fixed_details = matcher.match(fixed_symbols) - if not any([vuln_evidence, fixed_evidence]): + if not any([vuln_details, fixed_details]): continue report = { @@ -188,10 +188,10 @@ def collect_and_match_resources(self): "commit_hash": commit_hash, }, "advisory_uids": advisory_uids, - "evidence": list(vuln_evidence.values()), - "fixed_symbols": sorted(fixed_evidence.keys()), - "vulnerable_symbols": sorted(vuln_evidence.keys()), - "reachability_status": classify_reachability(vuln_evidence).value, + "tool_details": list(vuln_details.values()), + "fixed_symbols": sorted(fixed_details.keys()), + "vulnerable_symbols": sorted(vuln_details.keys()), + "is_reachable": classify_reachability(vuln_details).value, } add_reachability_report( @@ -223,40 +223,44 @@ def generate_advisory_reachability_report(self): ReachabilityStatus.NOT_REACHABLE.value: 1, } - advisory_reachability_report = {} + advisory_reachability_report = { + "purl": self.project.purl, + "advisories": [], + } + + advisory_map = {} for resource in self.candidate_resources: for report in resource.extra_data.get("symbols_reachability", []): advisory_uids = report.get("advisory_uids", []) - reachability_status = report.get("reachability_status") + is_reachable = report.get("is_reachable") patch = report.get("patch", {}) for adv_uid in advisory_uids: - if adv_uid not in advisory_reachability_report: - advisory_reachability_report[adv_uid] = { - "reachable": ReachabilityStatus.NOT_REACHABLE.value, + if adv_uid not in advisory_map: + adv_data = { + "advisory_uid": adv_uid, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, "details": [], } + advisory_map[adv_uid] = adv_data + advisory_reachability_report["advisories"].append(adv_data) tool_details = { "resource_path": resource.path, "patch": patch, - "reachability_status": reachability_status, - "evidence": report.get("evidence", []), + "is_reachable": is_reachable, + "tool_details": report.get("tool_details", []), "vulnerable_symbols": report.get("vulnerable_symbols", []), "fixed_symbols": report.get("fixed_symbols", []), } - advisory_reachability_report[adv_uid]["details"].append( - tool_details - ) + advisory_map[adv_uid]["details"].append(tool_details) - current_status = advisory_reachability_report[adv_uid]["reachable"] - if status_priority.get( - reachability_status, 0 - ) > status_priority.get(current_status, 0): - advisory_reachability_report[adv_uid]["reachable"] = ( - reachability_status - ) + current_status = advisory_map[adv_uid]["is_reachable"] + if status_priority.get(is_reachable, 0) > status_priority.get( + current_status, 0 + ): + advisory_map[adv_uid]["is_reachable"] = is_reachable reachability_output_path = self.project.get_output_file_path( "reachability", "json" diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index b4960d15bb..9e56d8c36e 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -36,9 +36,9 @@ class ReachabilityStatus(str, Enum): - REACHABLE = "YES" - UNKNOWN = "UNKNOWN" - NOT_REACHABLE = "NO" + REACHABLE = "yes" + UNKNOWN = "unknown" + NOT_REACHABLE = "no" def normalize_text(content): @@ -347,21 +347,21 @@ def analyze( return vuln_meta, fixed_meta, language -def classify_reachability(evidence): +def classify_reachability(tool_details): """ Classify the reachability status of a vulnerability based on the - collected evidence from :class:`ResourcePatchMatcher`. + collected tool_details from ResourcePatchMatcher. """ - if not evidence: + if not tool_details: return ReachabilityStatus.NOT_REACHABLE status = ReachabilityStatus.NOT_REACHABLE - for item in evidence.values(): - is_called = bool(item.get("called")) + for item in tool_details.values(): + is_called = bool(item.get("is_called")) has_path = bool(item.get("reachable_from")) - is_defined = bool(item.get("defined")) - is_imported = bool(item.get("imported")) - is_exact = bool(item.get("fingerprint")) + is_defined = bool(item.get("is_defined")) + is_imported = bool(item.get("is_imported")) + is_exact = bool(item.get("is_exact")) if is_exact or (is_imported and (is_called or has_path)): return ReachabilityStatus.REACHABLE @@ -461,16 +461,68 @@ def __init__(self, resource_index): self.imports = resource_index.get("imports", {}) self.callers_of = resource_index.get("callers_of", {}) self.separator = resource_index.get("separator", ".") + self.wildcard_modules = self.imports.get("*", []) + + def _matches_first_component( + self, qualified_name, abs_path, local_name, import_call_names + ): + """ + Check if the first component of qualified_name + matches the end of abs_path. + """ + first_component = qualified_name.split(self.separator, 1)[0] + if first_component != qualified_name and ( + abs_path.endswith(self.separator + first_component) + or abs_path == first_component + ): + remaining = qualified_name[len(first_component) :] + import_call_names.add(f"{local_name}{remaining}") + return True + return False + + def _matches_wildcard(self, qualified_name): + """Check if the qualified_name is covered by a wildcard import.""" + return any( + qualified_name == mod or qualified_name.startswith(mod + self.separator) + for mod in self.wildcard_modules + ) + + def _get_import_info(self, qualified_name): + """Check if qualified_name is imported and return possible call names.""" + import_call_names = set() + imported = False + + for local_name, abs_path in self.imports.items(): + if local_name == "*": + continue + + if qualified_name in (local_name, abs_path): + imported = True + import_call_names.add(local_name) + elif qualified_name.startswith(local_name + self.separator): + imported = True + import_call_names.add(qualified_name) + elif qualified_name.startswith(abs_path + self.separator): + imported = True + remaining = qualified_name[len(abs_path) :] + import_call_names.add(f"{local_name}{remaining}") + elif abs_path.endswith(self.separator + qualified_name): + imported = True + import_call_names.add(local_name) + elif self._matches_first_component( + qualified_name, abs_path, local_name, import_call_names + ): + imported = True + + if not imported and self._matches_wildcard(qualified_name): + return True, import_call_names + + return imported, import_call_names def match(self, patch_symbols_metadata): """ Match a set of patch symbols against the resource index and - return evidence for each matched symbol. - - For each symbol in patch_symbols_metadata, the method - checks whether it is defined, imported, called, or has an - exact fingerprint match in the resource. If at least one of - these conditions is true, an evidence entry is created. + return tool_details for each matched symbol. """ if not patch_symbols_metadata or not self.resource_index: return {} @@ -480,42 +532,56 @@ def match(self, patch_symbols_metadata): qualified_name = metadata["qualified_name"] fingerprint = metadata["fingerprint"] defined = qualified_name in self.definitions - fingerprint_hit = bool(fingerprint and fingerprint in self.fingerprints) - - imported = ( - qualified_name in self.imports - or qualified_name in self.imports.values() + is_exact = bool( + fingerprint + and fingerprint in self.fingerprints + and qualified_name in self.definitions ) + short_name = ( + qualified_name.rsplit(self.separator, 1)[-1] + if self.separator in qualified_name + else qualified_name + ) + + imported, import_call_names = self._get_import_info(qualified_name) - callers = set(self.callers_of.get(qualified_name, set())) - called = bool(callers) + possible_call_names = {qualified_name} + if imported or defined: + possible_call_names.add(short_name) + possible_call_names.update(import_call_names) - if not (defined or fingerprint_hit or called or imported): + callers = set() + for call_name in possible_call_names: + callers.update(self.callers_of.get(call_name, set())) + + called = bool(callers) and ( + imported or defined or bool(self.wildcard_modules) + ) + if called and not imported and not defined and self.wildcard_modules: + imported = True + + if not (defined or is_exact or called or imported): continue entry = matched.setdefault( qualified_name, { "symbol_name": qualified_name, - "called": False, - "defined": False, - "imported": False, - "fingerprint": None, + "is_called": False, + "is_defined": False, + "is_imported": False, + "is_exact": False, "reachable_from": [], }, ) - if defined: - entry["defined"] = True - if imported: - entry["imported"] = True + entry["is_defined"] = entry["is_defined"] or defined + entry["is_imported"] = entry["is_imported"] or imported + entry["is_exact"] = entry["is_exact"] or is_exact + entry["is_called"] = entry["is_called"] or called if called: - entry["called"] = True entry["reachable_from"] = sorted(callers) - if fingerprint_hit: - entry["fingerprint"] = fingerprint - return matched diff --git a/scanpipe/pipes/symbols.py b/scanpipe/pipes/symbols.py index 8abd989bdb..3bd97694f2 100644 --- a/scanpipe/pipes/symbols.py +++ b/scanpipe/pipes/symbols.py @@ -377,7 +377,12 @@ class JavaTreeSitterQuery(LanguageQuery): (import_declaration (scoped_identifier) @import_name) (import_declaration (scoped_identifier) @module_name (asterisk) @import_name) """ - syntax_config = {"self_keyword": "this", "separator": ".", "wildcard_symbol": "*"} + syntax_config = { + "self_keyword": "this", + "separator": ".", + "wildcard_symbol": "*", + "import_local_name": "last", + } TS_QUERIES = { @@ -469,9 +474,29 @@ def extract_calls(self, node): return calls + def resolve_local_name(self, imp_name, alias): + """Resolve the local name for an imported symbol.""" + separator = self.syntax_config.get("separator", ".") + local_name = alias or imp_name + + if not alias and separator in imp_name: + if self.syntax_config.get("import_local_name") == "last": + return imp_name.split(separator)[-1] + return imp_name.split(separator)[0] + + return local_name + + def resolve_absolute_path(self, module_name, imp_name): + """Resolve the absolute path for an imported symbol.""" + separator = self.syntax_config.get("separator", ".") + if module_name: + if module_name == separator: + return f"{separator}{imp_name}" + return f"{module_name}{separator}{imp_name}" + return imp_name + def extract_imports(self): """Map every local alias to its absolute imported path.""" - separator = self.syntax_config.get("separator", ".") wildcard_sym = self.syntax_config.get("wildcard_symbol") import_map = {} @@ -487,17 +512,8 @@ def extract_imports(self): wildcard_modules.append(module_name) continue - local_name = alias or imp_name - if not alias and separator in imp_name: - local_name = imp_name.split(separator)[0] - - if module_name: - if module_name == separator: - absolute_path = f"{separator}{imp_name}" - else: - absolute_path = f"{module_name}{separator}{imp_name}" - else: - absolute_path = imp_name + local_name = self.resolve_local_name(imp_name, alias) + absolute_path = self.resolve_absolute_path(module_name, imp_name) import_map[local_name] = absolute_path diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 88f51c9b5b..5002441b46 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -37,6 +37,8 @@ from scanpipe.pipes import collect_and_create_codebase_resources from scanpipe.pipes.reachability import PatchAnalyzer from scanpipe.pipes.reachability import ReachabilityStatus +from scanpipe.pipes.reachability import ResourceAnalyzer +from scanpipe.pipes.reachability import ResourcePatchMatcher from scanpipe.pipes.reachability import classify_reachability from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor @@ -121,21 +123,20 @@ def test_end_to_end_symbol_reachability_pipeline( "vcs_url": repo_url, "commit_hash": fixed_commit.hexsha, }, - "advisory_uids": [], - "evidence": [ + "is_reachable": ReachabilityStatus.REACHABLE.value, + "tool_details": [ { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "e66988810af452f77d43efd302fea4c" - "1d2f246d282c4ea8982b0152c2231ac17", + "is_exact": True, + "is_called": False, + "is_defined": True, + "is_imported": False, "symbol_name": "process_data", "reachable_from": [], } ], + "advisory_uids": [], "fixed_symbols": ["process_data"], "vulnerable_symbols": ["process_data"], - "reachability_status": ReachabilityStatus.REACHABLE.value, } ] @@ -218,6 +219,7 @@ def test_generate_advisory_reachability_report(self): res_empty = MagicMock() res_empty.path = "src/empty.py" + res_empty.purl = "pkg:pypi/test" res_empty.extra_data = {} pipeline.candidate_resources = [res2, res3, res_empty, res1] @@ -227,80 +229,65 @@ def test_generate_advisory_reachability_report(self): report = json.load(f) expected_report = { - "AVID-1": { - "reachable": ReachabilityStatus.REACHABLE.value, - "details": [ - { - "resource_path": "src/file2.py", - "patch": { - "vcs_url": "https://example.com", - "commit_hash": "def456", + "purl": "", + "advisories": [ + { + "advisory_uid": "AVID-1", + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, + "details": [ + { + "resource_path": "src/file2.py", + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "def456", + }, + "is_reachable": None, + "tool_details": [], + "vulnerable_symbols": [], + "fixed_symbols": [], }, - "reachability_status": "UNKNOWN", - "evidence": [], - "vulnerable_symbols": [], - "fixed_symbols": [], - }, - { - "resource_path": "src/file1.py", - "patch": { - "vcs_url": "https://example.com", - "commit_hash": "abc123", + { + "resource_path": "src/file1.py", + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "abc123", + }, + "is_reachable": None, + "tool_details": [], + "vulnerable_symbols": ["vuln_sym1"], + "fixed_symbols": ["fixed_sym1"], }, - "reachability_status": ReachabilityStatus.REACHABLE.value, - "evidence": [ - { - "symbol_name": "vuln_sym1", - "called": True, - "defined": False, - "imported": False, - "fingerprint": "88ad9e67c53aa5f7c43e" - "c4aa52ed34b7930068c9", - "reachable_from": [], - } - ], - "vulnerable_symbols": ["vuln_sym1"], - "fixed_symbols": ["fixed_sym1"], - }, - ], - }, - "AVID-2": { - "reachable": ReachabilityStatus.REACHABLE.value, - "details": [ - { - "resource_path": "src/file3.py", - "patch": { - "vcs_url": "https://example2.com", - "commit_hash": "46a4asf", + ], + }, + { + "advisory_uid": "AVID-2", + "is_reachable": "no", + "details": [ + { + "resource_path": "src/file3.py", + "patch": { + "vcs_url": "https://example2.com", + "commit_hash": "46a4asf", + }, + "is_reachable": None, + "tool_details": [], + "vulnerable_symbols": [], + "fixed_symbols": [], }, - "reachability_status": "NO", - "evidence": [], - "vulnerable_symbols": [], - "fixed_symbols": [], - }, - { - "resource_path": "src/file1.py", - "patch": { - "vcs_url": "https://example.com", - "commit_hash": "abc123", + { + "resource_path": "src/file1.py", + "patch": { + "vcs_url": "https://example.com", + "commit_hash": "abc123", + }, + "is_reachable": None, + "tool_details": [], + "vulnerable_symbols": ["vuln_sym1"], + "fixed_symbols": ["fixed_sym1"], }, - "reachability_status": ReachabilityStatus.REACHABLE.value, - "evidence": [ - { - "symbol_name": "vuln_sym1", - "called": True, - "defined": False, - "imported": False, - "fingerprint": "88ad9e67c53aa5f7c4" - "3ec4aa52ed34b7930068c9", - "reachable_from": [], - } - ], - "vulnerable_symbols": ["vuln_sym1"], - "fixed_symbols": ["fixed_sym1"], - }, - ], - }, + ], + }, + ], } self.assertEqual(report, expected_report) @@ -395,36 +382,34 @@ def test_python_get_symbol_reachability_results( "vcs_url": "https://github.com/aboutcode-org/test", "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", }, - "advisory_uids": [], - "evidence": [ + "is_reachable": "yes", + "tool_details": [ { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "336908735214468b103dbde" - "11c3ffbd2f76ac9212b8514f831cfa078a67892df", + "is_exact": True, + "is_called": False, + "is_defined": True, + "is_imported": False, "symbol_name": "debug", "reachable_from": [], }, { - "called": False, - "defined": True, - "imported": False, - "fingerprint": "762e4f7d03b1bf4359c3ca364" - "e558140239913bfabcc5aa77156460c2eb0a355", + "is_exact": True, + "is_called": True, + "is_defined": True, + "is_imported": False, "symbol_name": "serve_report.build_file_path", - "reachable_from": [], + "reachable_from": ["serve_report"], }, { - "called": True, - "defined": True, - "imported": False, - "fingerprint": "d7675efb263896da2a3c0067951183" - "3553907e7e6ea619115a6dfc8625c3457e", + "is_exact": True, + "is_called": True, + "is_defined": True, + "is_imported": False, "symbol_name": "serve_report", "reachable_from": ["handle_request"], }, ], + "advisory_uids": [], "fixed_symbols": [ "debug", "serve_report", @@ -435,7 +420,6 @@ def test_python_get_symbol_reachability_results( "serve_report", "serve_report.build_file_path", ], - "reachability_status": ReachabilityStatus.REACHABLE.value, } ] @@ -471,44 +455,36 @@ def test_java_get_symbol_reachability_results( "vcs_url": "https://github.com/aboutcode-org/test", "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", }, - "advisory_uids": [], - "evidence": [ + "is_reachable": "yes", + "tool_details": [ { + "is_exact": False, + "is_called": False, + "is_defined": True, + "is_imported": False, "symbol_name": "App", - "called": False, - "defined": True, - "imported": False, - "fingerprint": None, "reachable_from": [], }, { + "is_exact": True, + "is_called": False, + "is_defined": True, + "is_imported": False, "symbol_name": "App.serveReport", - "called": False, - "defined": True, - "imported": False, - "fingerprint": "b161e24e9575b655e84c7f249709e5d4d0a1e6f19e2c4baa421a6cc996fda154", - "reachable_from": [] + "reachable_from": [], }, { + "is_exact": False, + "is_called": True, + "is_defined": True, + "is_imported": False, "symbol_name": "App.buildFilePath", - "called": False, - "defined": True, - "imported": False, - "fingerprint": None, - "reachable_from": [] - } - ], - "fixed_symbols": [ - "App", - "App.buildFilePath", - "App.serveReport" - ], - "vulnerable_symbols": [ - "App", - "App.buildFilePath", - "App.serveReport" + "reachable_from": ["App.serveReport"], + }, ], - "reachability_status": ReachabilityStatus.REACHABLE.value, + "advisory_uids": [], + "fixed_symbols": ["App", "App.buildFilePath", "App.serveReport"], + "vulnerable_symbols": ["App", "App.buildFilePath", "App.serveReport"], } ] @@ -631,23 +607,30 @@ def test_classify_reachability(self): self.assertEqual(classify_reachability(None), ReachabilityStatus.NOT_REACHABLE) self.assertEqual(classify_reachability({}), ReachabilityStatus.NOT_REACHABLE) self.assertEqual( - classify_reachability({"evidence": {}}), ReachabilityStatus.NOT_REACHABLE + classify_reachability({"tool_details": {}}), + ReachabilityStatus.NOT_REACHABLE, ) self.assertEqual( - classify_reachability({"evidence": {"fingerprint": "hash123"}}), + classify_reachability({"tool_details": {"is_exact": True}}), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"evidence": {"imported": True, "called": True}}), + classify_reachability( + {"tool_details": {"is_imported": True, "is_called": True}} + ), ReachabilityStatus.REACHABLE, ) self.assertEqual( - classify_reachability({"evidence": {"imported": True, "called": False}}), + classify_reachability( + {"tool_details": {"is_imported": True, "is_called": False}} + ), ReachabilityStatus.UNKNOWN, ) self.assertEqual( - classify_reachability({"symbol1": {"imported": False, "called": False}}), + classify_reachability( + {"tool_details": {"is_imported": False, "is_called": False}} + ), ReachabilityStatus.NOT_REACHABLE, ) @@ -1032,3 +1015,231 @@ def test_extract_imports(self): } self.assertEqual(result, expected_map) + + def test_resource_patch_matcher_python(self): + """ + Test matching + Python patch symbols against imports and calls. + """ + vuln_text = """ +def direct_func(): + return eval("1") + +def aliased_func(): + return eval("2") + +def wildcard_func(): + return eval("3") + +class MyClass: + def class_method(self): + return eval("4") + + class InnerClass: + def deep_method(self): + return eval("5") +""".strip() + + fixed_text = """ +def direct_func(): + return int("1") + +def aliased_func(): + return int("2") + +def wildcard_func(): + return int("3") + +class MyClass: + def class_method(self): + return int("4") + + class InnerClass: + def deep_method(self): + return int("5") +""".strip() + + app_text = """ +from my_module import direct_func +from my_module import aliased_func as af +from my_module import MyClass as C +from my_module import wildcard_func +from my_module import * + +def execute(): + direct_func() + af() + wildcard_func() + + c = C() + c.class_method() + + inner = C.InnerClass() + inner.deep_method() +""".strip() + + file_path = "my_module.py" + analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") + removed_lines, added_lines = analyzer.compute_changed_lines( + vuln_text, fixed_text + ) + + vuln_meta, fixed_meta, lang = analyzer.analyze( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, + ) + + patch_symbols_by_language = { + lang: { + "vulnerable": { + f"{file_path}::{key}": metadata + for key, metadata in vuln_meta.items() + }, + "fixed": { + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() + }, + } + } + + resource_analyzer = ResourceAnalyzer(resource_text=app_text, language=lang) + resource_index = resource_analyzer.build_index() + matcher = ResourcePatchMatcher(resource_index) + + vulnerable_symbols = patch_symbols_by_language[lang]["vulnerable"] + fixed_symbols = patch_symbols_by_language[lang]["fixed"] + + vuln_details = matcher.match(vulnerable_symbols) + fixed_details = matcher.match(fixed_symbols) + + matched = {**vuln_details, **fixed_details} + self.assertIn("direct_func", matched) + self.assertTrue(matched["direct_func"]["is_imported"]) + self.assertTrue(matched["direct_func"]["is_called"]) + self.assertEqual(matched["direct_func"]["reachable_from"], ["execute"]) + + self.assertIn("aliased_func", matched) + self.assertTrue(matched["aliased_func"]["is_imported"]) + self.assertTrue(matched["aliased_func"]["is_called"]) + self.assertEqual(matched["aliased_func"]["reachable_from"], ["execute"]) + + self.assertIn("wildcard_func", matched) + self.assertTrue(matched["wildcard_func"]["is_imported"]) + self.assertTrue(matched["wildcard_func"]["is_called"]) + self.assertEqual(matched["wildcard_func"]["reachable_from"], ["execute"]) + + self.assertIn("MyClass.class_method", matched) + self.assertTrue(matched["MyClass.class_method"]["is_imported"]) + self.assertTrue(matched["MyClass.class_method"]["is_called"]) + self.assertEqual(matched["MyClass.class_method"]["reachable_from"], ["execute"]) + + self.assertIn("MyClass.InnerClass.deep_method", matched) + self.assertTrue(matched["MyClass.InnerClass.deep_method"]["is_imported"]) + self.assertTrue(matched["MyClass.InnerClass.deep_method"]["is_called"]) + self.assertEqual( + matched["MyClass.InnerClass.deep_method"]["reachable_from"], ["execute"] + ) + + def test_resource_patch_matcher_java(self): + """ + Test matching + Java patch symbols against imports and calls. + """ + vuln_text = """ +package com.example; + +public class Service { + public void processRequest(String input) { + Runtime.getRuntime().exec(input); + } + + public void utilityMethod() { + Runtime.getRuntime().exec("cmd"); + } +} + """.strip() + + fixed_text = """ +package com.example; + +public class Service { + public void processRequest(String input) { + System.out.println(input); + } + + public void utilityMethod() { + System.out.println("cmd"); + } +} + """.strip() + + app_text = """ +package com.test; + +import com.example.Service; +import com.example.*; + +public class Main { + public void execute(String data) { + Service service = new Service(); + service.processRequest(data); + + service.utilityMethod(); + } +} + """.strip() + + file_path = "Service.java" + analyzer = PatchAnalyzer(repo=MagicMock(), commit_hash="dummy") + removed_lines, added_lines = analyzer.compute_changed_lines( + vuln_text, fixed_text + ) + + vuln_meta, fixed_meta, lang = analyzer.analyze( + vulnerable_text=vuln_text, + fixed_text=fixed_text, + removed_lines=removed_lines, + added_lines=added_lines, + file_path=file_path, + ) + + patch_symbols_by_language = { + lang: { + "vulnerable": { + f"{file_path}::{key}": metadata + for key, metadata in vuln_meta.items() + }, + "fixed": { + f"{file_path}::{key}": metadata + for key, metadata in fixed_meta.items() + }, + } + } + + resource_analyzer = ResourceAnalyzer(resource_text=app_text, language=lang) + resource_index = resource_analyzer.build_index() + matcher = ResourcePatchMatcher(resource_index) + + vulnerable_symbols = patch_symbols_by_language[lang]["vulnerable"] + fixed_symbols = patch_symbols_by_language[lang]["fixed"] + + vuln_details = matcher.match(vulnerable_symbols) + fixed_details = matcher.match(fixed_symbols) + + matched = {**vuln_details, **fixed_details} + self.assertIn("Service.processRequest", matched) + self.assertTrue(matched["Service.processRequest"]["is_imported"]) + self.assertTrue(matched["Service.processRequest"]["is_called"]) + self.assertEqual( + matched["Service.processRequest"]["reachable_from"], ["Main.execute"] + ) + + self.assertIn("Service.utilityMethod", matched) + self.assertTrue(matched["Service.utilityMethod"]["is_imported"]) + self.assertTrue(matched["Service.utilityMethod"]["is_called"]) + self.assertEqual( + matched["Service.utilityMethod"]["reachable_from"], ["Main.execute"] + ) From 30c8c7b9034bc0568a1eceadbe034414c4058f30 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 24 Aug 2026 05:36:47 +0300 Subject: [PATCH 40/50] Add more test and make sure it returns not if there is no match Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 11 ++++ .../tests/pipes/test_symbols_reachability.py | 60 ++++++++++++++++--- 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 186ea7345e..c0a39916a6 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -229,6 +229,17 @@ def generate_advisory_reachability_report(self): } advisory_map = {} + for patch in self.patches: + for adv_uid in patch.get("advisory_uids", []): + if adv_uid not in advisory_map: + adv_data = { + "advisory_uid": adv_uid, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, + "details": [], + } + advisory_map[adv_uid] = adv_data + advisory_reachability_report["advisories"].append(adv_data) + for resource in self.candidate_resources: for report in resource.extra_data.get("symbols_reachability", []): advisory_uids = report.get("advisory_uids", []) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 5002441b46..d1fb6853b0 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -149,6 +149,12 @@ def test_generate_advisory_reachability_report(self): run = self.project1.add_pipeline("analyze_symbols_reachability") pipeline = run.make_pipeline_instance() + pipeline.patches = [ + {"advisory_uids": ["AVID-1"]}, + {"advisory_uids": ["AVID-2"]}, + ] + pipeline.project.purl = "pkg:pypi/test" + with tempfile.TemporaryDirectory() as tmpdir: output_file = os.path.join(tmpdir, "reachability.json") pipeline.project.get_output_file_path = MagicMock(return_value=output_file) @@ -219,7 +225,6 @@ def test_generate_advisory_reachability_report(self): res_empty = MagicMock() res_empty.path = "src/empty.py" - res_empty.purl = "pkg:pypi/test" res_empty.extra_data = {} pipeline.candidate_resources = [res2, res3, res_empty, res1] @@ -229,7 +234,7 @@ def test_generate_advisory_reachability_report(self): report = json.load(f) expected_report = { - "purl": "", + "purl": "pkg:pypi/test", "advisories": [ { "advisory_uid": "AVID-1", @@ -1031,14 +1036,23 @@ def aliased_func(): def wildcard_func(): return eval("3") +def multiline_func(): + return eval("6") + +def deep_func(): + return eval("7") + +def relative_func(): + return eval("8") + class MyClass: def class_method(self): return eval("4") - class InnerClass: - def deep_method(self): - return eval("5") -""".strip() +class InnerClass: + def deep_method(self): + return eval("5") + """.strip() fixed_text = """ def direct_func(): @@ -1050,6 +1064,15 @@ def aliased_func(): def wildcard_func(): return int("3") +def multiline_func(): + return int("6") + +def deep_func(): + return int("7") + +def relative_func(): + return int("8") + class MyClass: def class_method(self): return int("4") @@ -1057,7 +1080,7 @@ def class_method(self): class InnerClass: def deep_method(self): return int("5") -""".strip() + """.strip() app_text = """ from my_module import direct_func @@ -1065,11 +1088,19 @@ def deep_method(self): from my_module import MyClass as C from my_module import wildcard_func from my_module import * +from a.b import deep_func +from .relative_module import relative_func +from my_module import ( + multiline_func, +) def execute(): direct_func() af() wildcard_func() + multiline_func() + deep_func() + relative_func() c = C() c.class_method() @@ -1143,6 +1174,21 @@ def execute(): matched["MyClass.InnerClass.deep_method"]["reachable_from"], ["execute"] ) + self.assertIn("multiline_func", matched) + self.assertTrue(matched["multiline_func"]["is_imported"]) + self.assertTrue(matched["multiline_func"]["is_called"]) + self.assertEqual(matched["multiline_func"]["reachable_from"], ["execute"]) + + self.assertIn("deep_func", matched) + self.assertTrue(matched["deep_func"]["is_imported"]) + self.assertTrue(matched["deep_func"]["is_called"]) + self.assertEqual(matched["deep_func"]["reachable_from"], ["execute"]) + + self.assertIn("relative_func", matched) + self.assertTrue(matched["relative_func"]["is_imported"]) + self.assertTrue(matched["relative_func"]["is_called"]) + self.assertEqual(matched["relative_func"]["reachable_from"], ["execute"]) + def test_resource_patch_matcher_java(self): """ Test matching From fa64a683e55822c9a74079d48d6e44f9a0b27fc6 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Mon, 24 Aug 2026 16:59:20 +0300 Subject: [PATCH 41/50] Move Business logic to be in the pipes reachability file Fix the test Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 226 ++---------------- scanpipe/pipes/reachability.py | 212 +++++++++++++++- .../tests/pipes/test_symbols_reachability.py | 4 +- 3 files changed, 234 insertions(+), 208 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index c0a39916a6..06c2d6318f 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -20,19 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -import json -import tempfile - -from git import Repo - from scanpipe.pipelines import Pipeline -from scanpipe.pipes.reachability import PatchAnalyzer -from scanpipe.pipes.reachability import ReachabilityStatus -from scanpipe.pipes.reachability import ResourceAnalyzer -from scanpipe.pipes.reachability import ResourcePatchMatcher -from scanpipe.pipes.reachability import add_reachability_report -from scanpipe.pipes.reachability import classify_reachability -from scanpipe.pipes.reachability import normalize_text +from scanpipe.pipes import reachability class SymbolReachability(Pipeline): @@ -71,211 +60,40 @@ def steps(cls): def get_vulnerabilities_patches(self): """Get unique patch for all vulnerabilities.""" - patches = {} - for vulnerability in ( - self.project.package_vulnerabilities - + self.project.dependency_vulnerabilities - ): - advisory_uid = vulnerability.get("advisory_uid") - for patch in vulnerability.get("fixed_in_patches", []): - vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - - p_key = (vcs_url, commit_hash) - if p_key not in patches: - patches[p_key] = { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - "advisory_uids": [], - } - - if advisory_uid and advisory_uid not in patches[p_key]["advisory_uids"]: - patches[p_key]["advisory_uids"].append(advisory_uid) - - self.patches = list(patches.values()) + self.patches = reachability.get_vulnerabilities_patches( + package_vulnerabilities=self.project.package_vulnerabilities, + dependency_vulnerabilities=self.project.dependency_vulnerabilities, + ) def collect_resource_index(self): """Collect resources symbols for each resource exactly once.""" self.candidate_resources = self.project.codebaseresources.files().filter( is_binary=False, is_archive=False, is_media=False ) - self.resource_indexes = {} - for resource in self.candidate_resources: - resource_language = resource.programming_language - - file_content = normalize_text(resource.file_content) - if not file_content: - continue - - resource_analyzer = ResourceAnalyzer( - resource_text=file_content, language=resource_language - ) - - resource_index = resource_analyzer.build_index() - if resource_index: - self.resource_indexes[resource.path] = resource_index + self.resource_indexes = reachability.collect_resource_index( + candidate_resources=self.candidate_resources + ) def collect_patch_symbols(self): """ - For each unique repo: clone it once, collect patch symbols for all - related commits, then remove the local clone before moving on + Collect patch symbols for all related commits, + then remove the local clone before moving on """ - self.patch_symbols = {} - patches_by_repo = {} - for patch in self.patches: - vcs_url = patch.get("vcs_url") - patches_by_repo.setdefault(vcs_url, []).append(patch) - - for vcs_url, repo_patches in patches_by_repo.items(): - with tempfile.TemporaryDirectory( - prefix="symbol-reachability-" - ) as repo_path: - try: - repo = Repo.clone_from(vcs_url, repo_path) - except Exception as e: - raise Exception(f"Failed to clone repository {vcs_url}: {e!r}") - - try: - for patch in repo_patches: - commit_hash = patch.get("commit_hash") - patch_analyzer = PatchAnalyzer( - repo=repo, commit_hash=commit_hash - ) - self.patch_symbols[commit_hash] = ( - patch_analyzer.collect_patch_symbols() - ) - except Exception as e: - raise Exception( - f"Failed to collect patch symbols " - f"for {vcs_url}, patch: {repo_patches}: {e!r}" - ) + self.patch_symbols = reachability.collect_patch_symbols(patches=self.patches) def collect_and_match_resources(self): """Match resource symbols against patch symbols.""" - for patch in self.patches: - vcs_url = patch.get("vcs_url") - commit_hash = patch.get("commit_hash") - advisory_uids = patch.get("advisory_uids", []) - - patch_symbols_by_language = self.patch_symbols.get(commit_hash, {}) - if not patch_symbols_by_language: - continue - - for resource in self.candidate_resources: - resource_index = self.resource_indexes.get(resource.path) - if not resource_index: - continue - - patch_symbols = patch_symbols_by_language.get( - resource.programming_language - ) - if not patch_symbols: - continue - - vulnerable_symbols = patch_symbols.get("vulnerable", {}) - fixed_symbols = patch_symbols.get("fixed", {}) - - matcher = ResourcePatchMatcher(resource_index=resource_index) - vuln_details = matcher.match(vulnerable_symbols) - fixed_details = matcher.match(fixed_symbols) - - if not any([vuln_details, fixed_details]): - continue - - report = { - "patch": { - "vcs_url": vcs_url, - "commit_hash": commit_hash, - }, - "advisory_uids": advisory_uids, - "tool_details": list(vuln_details.values()), - "fixed_symbols": sorted(fixed_details.keys()), - "vulnerable_symbols": sorted(vuln_details.keys()), - "is_reachable": classify_reachability(vuln_details).value, - } - - add_reachability_report( - resource=resource, - commit_hash=commit_hash, - vcs_url=vcs_url, - new_report=report, - ) + reachability.collect_and_match_resources( + patches=self.patches, + patch_symbols=self.patch_symbols, + resource_indexes=self.resource_indexes, + candidate_resources=self.candidate_resources, + ) def generate_advisory_reachability_report(self): - """ - Generate reachability report keyed by advisory_uid. - - Each advisory contains its overall reachability status - and the reachability results collected from all resources - and associated patches. - - The overall reachability status is determined using the following - priority order: REACHABLE > UNKNOWN > NOT_REACHABLE - - This means that an advisory is considered reachable if it is reachable - through at least one resource or patch. If no reachable result exists, - but at least one result is unknown, the advisory status is UNKNOWN. - Otherwise, it is NOT_REACHABLE. - """ - status_priority = { - ReachabilityStatus.REACHABLE.value: 3, - ReachabilityStatus.UNKNOWN.value: 2, - ReachabilityStatus.NOT_REACHABLE.value: 1, - } - - advisory_reachability_report = { - "purl": self.project.purl, - "advisories": [], - } - - advisory_map = {} - for patch in self.patches: - for adv_uid in patch.get("advisory_uids", []): - if adv_uid not in advisory_map: - adv_data = { - "advisory_uid": adv_uid, - "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, - "details": [], - } - advisory_map[adv_uid] = adv_data - advisory_reachability_report["advisories"].append(adv_data) - - for resource in self.candidate_resources: - for report in resource.extra_data.get("symbols_reachability", []): - advisory_uids = report.get("advisory_uids", []) - is_reachable = report.get("is_reachable") - patch = report.get("patch", {}) - - for adv_uid in advisory_uids: - if adv_uid not in advisory_map: - adv_data = { - "advisory_uid": adv_uid, - "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, - "details": [], - } - advisory_map[adv_uid] = adv_data - advisory_reachability_report["advisories"].append(adv_data) - - tool_details = { - "resource_path": resource.path, - "patch": patch, - "is_reachable": is_reachable, - "tool_details": report.get("tool_details", []), - "vulnerable_symbols": report.get("vulnerable_symbols", []), - "fixed_symbols": report.get("fixed_symbols", []), - } - - advisory_map[adv_uid]["details"].append(tool_details) - - current_status = advisory_map[adv_uid]["is_reachable"] - if status_priority.get(is_reachable, 0) > status_priority.get( - current_status, 0 - ): - advisory_map[adv_uid]["is_reachable"] = is_reachable - - reachability_output_path = self.project.get_output_file_path( - "reachability", "json" + """Generate reachability report keyed by advisory_uid.""" + reachability.generate_advisory_reachability_report( + project=self.project, + patches=self.patches, + candidate_resources=self.candidate_resources, ) - - with open(reachability_output_path, "w") as f: - json.dump(advisory_reachability_report, f, indent=2) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 9e56d8c36e..e6353f121f 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -21,11 +21,13 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. import difflib +import json import shutil import tempfile from enum import Enum from pathlib import Path +from git import Repo from git.diff import NULL_TREE from scancode.api import get_file_info @@ -585,9 +587,9 @@ def match(self, patch_symbols_metadata): return matched -def add_reachability_report(resource, commit_hash, vcs_url, new_report): +def save_resource_reachability_report(resource, commit_hash, vcs_url, new_report): """ - Add a reachability report for commit_hash and vcs_url. + Save a reachability report for commit_hash and vcs_url. If duplicates exist, replace the old report with the new one. """ cleaned_reports = [] @@ -611,3 +613,209 @@ def add_reachability_report(resource, commit_hash, vcs_url, new_report): cleaned_reports.append(new_report) resource.update_extra_data({"symbols_reachability": cleaned_reports}) + + +def get_vulnerabilities_patches(package_vulnerabilities, dependency_vulnerabilities): + """Get unique patch for all vulnerabilities.""" + patches = {} + for vulnerability in package_vulnerabilities + dependency_vulnerabilities: + advisory_uid = vulnerability.get("advisory_uid") + for patch in vulnerability.get("fixed_in_patches", []): + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + + p_key = (vcs_url, commit_hash) + if p_key not in patches: + patches[p_key] = { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + "advisory_uids": [], + } + + if advisory_uid and advisory_uid not in patches[p_key]["advisory_uids"]: + patches[p_key]["advisory_uids"].append(advisory_uid) + + return list(patches.values()) + + +def collect_resource_index(candidate_resources): + """Collect resources symbols for each resource exactly once.""" + resource_indexes = {} + for resource in candidate_resources: + resource_language = resource.programming_language + + file_content = normalize_text(resource.file_content) + if not file_content: + continue + + resource_analyzer = ResourceAnalyzer( + resource_text=file_content, language=resource_language + ) + + resource_index = resource_analyzer.build_index() + if resource_index: + resource_indexes[resource.path] = resource_index + + return resource_indexes + + +def collect_patch_symbols(patches): + """ + For each unique repo: clone it once, collect patch symbols for all + related commits, then remove the local clone before moving on + """ + patch_symbols = {} + patches_by_repo = {} + for patch in patches: + vcs_url = patch.get("vcs_url") + patches_by_repo.setdefault(vcs_url, []).append(patch) + + for vcs_url, repo_patches in patches_by_repo.items(): + with tempfile.TemporaryDirectory(prefix="symbol-reachability-") as repo_path: + try: + repo = Repo.clone_from(vcs_url, repo_path) + except Exception as e: + raise Exception(f"Failed to clone repository {vcs_url}: {e!r}") + + try: + for patch in repo_patches: + commit_hash = patch.get("commit_hash") + patch_analyzer = PatchAnalyzer(repo=repo, commit_hash=commit_hash) + patch_symbols[commit_hash] = patch_analyzer.collect_patch_symbols() + except Exception as e: + raise Exception( + f"Failed to collect patch symbols " + f"for {vcs_url}, patch: {repo_patches}: {e!r}" + ) + + return patch_symbols + + +def collect_and_match_resources( + patches, patch_symbols, candidate_resources, resource_indexes +): + """Match resource symbols against patch symbols.""" + for patch in patches: + vcs_url = patch.get("vcs_url") + commit_hash = patch.get("commit_hash") + advisory_uids = patch.get("advisory_uids", []) + + patch_symbols_by_language = patch_symbols.get(commit_hash, {}) + if not patch_symbols_by_language: + continue + + for resource in candidate_resources: + resource_index = resource_indexes.get(resource.path) + if not resource_index: + continue + + patch_symbols = patch_symbols_by_language.get(resource.programming_language) + if not patch_symbols: + continue + + vulnerable_symbols = patch_symbols.get("vulnerable", {}) + fixed_symbols = patch_symbols.get("fixed", {}) + + matcher = ResourcePatchMatcher(resource_index=resource_index) + vuln_details = matcher.match(vulnerable_symbols) + fixed_details = matcher.match(fixed_symbols) + + if not any([vuln_details, fixed_details]): + continue + + report = { + "patch": { + "vcs_url": vcs_url, + "commit_hash": commit_hash, + }, + "advisory_uids": advisory_uids, + "tool_details": list(vuln_details.values()), + "fixed_symbols": sorted(fixed_details.keys()), + "vulnerable_symbols": sorted(vuln_details.keys()), + "is_reachable": classify_reachability(vuln_details).value, + } + + save_resource_reachability_report( + resource=resource, + commit_hash=commit_hash, + vcs_url=vcs_url, + new_report=report, + ) + + +def generate_advisory_reachability_report(project, patches, candidate_resources): + """ + Generate reachability report keyed by advisory_uid. + + Each advisory contains its overall reachability status + and the reachability results collected from all resources + and associated patches. + + The overall reachability status is determined using the following + priority order: REACHABLE > UNKNOWN > NOT_REACHABLE + + This means that an advisory is considered reachable if it is reachable + through at least one resource or patch. If no reachable result exists, + but at least one result is unknown, the advisory status is UNKNOWN. + Otherwise, it is NOT_REACHABLE. + """ + status_priority = { + ReachabilityStatus.REACHABLE.value: 3, + ReachabilityStatus.UNKNOWN.value: 2, + ReachabilityStatus.NOT_REACHABLE.value: 1, + } + + advisory_reachability_report = { + "purl": project.purl, + "advisories": [], + } + + advisory_map = {} + for patch in patches: + for adv_uid in patch.get("advisory_uids", []): + if adv_uid not in advisory_map: + adv_data = { + "advisory_uid": adv_uid, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, + "details": [], + } + advisory_map[adv_uid] = adv_data + advisory_reachability_report["advisories"].append(adv_data) + + for resource in candidate_resources: + for report in resource.extra_data.get("symbols_reachability", []): + advisory_uids = report.get("advisory_uids", []) + is_reachable = report.get("is_reachable") + patch = report.get("patch", {}) + + for adv_uid in advisory_uids: + if adv_uid not in advisory_map: + adv_data = { + "advisory_uid": adv_uid, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, + "details": [], + } + advisory_map[adv_uid] = adv_data + advisory_reachability_report["advisories"].append(adv_data) + + tool_details = { + "resource_path": resource.path, + "patch": patch, + "is_reachable": is_reachable, + "tool_details": report.get("tool_details", []), + "vulnerable_symbols": report.get("vulnerable_symbols", []), + "fixed_symbols": report.get("fixed_symbols", []), + } + + advisory_map[adv_uid]["details"].append(tool_details) + + current_status = advisory_map[adv_uid]["is_reachable"] + if status_priority.get(is_reachable, 0) > status_priority.get( + current_status, 0 + ): + advisory_map[adv_uid]["is_reachable"] = is_reachable + + reachability_output_path = project.get_output_file_path("reachability", "json") + + with open(reachability_output_path, "w") as f: + json.dump(advisory_reachability_report, f, indent=2) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index d1fb6853b0..7afdd93dad 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -366,7 +366,7 @@ def _run_reachability_pipeline( results = resource.extra_data.get("symbols_reachability") self.assertEqual(results, expected_results) - @patch("scanpipe.pipelines.analyze_symbols_reachability.Repo") + @patch("scanpipe.pipes.reachability.Repo") @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_python_get_symbol_reachability_results( @@ -439,7 +439,7 @@ def test_python_get_symbol_reachability_results( expected_results, ) - @patch("scanpipe.pipelines.analyze_symbols_reachability.Repo") + @patch("scanpipe.pipes.reachability.Repo") @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) def test_java_get_symbol_reachability_results( From 8f83785206962456107069833e1ff59f49572349 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 25 Aug 2026 05:03:21 +0300 Subject: [PATCH 42/50] fix is_reachable to be by default NOT_REACHABLE Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 4 +++- scanpipe/tests/pipes/test_symbols_reachability.py | 8 ++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index e6353f121f..9d3cf3f9ad 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -785,7 +785,9 @@ def generate_advisory_reachability_report(project, patches, candidate_resources) for resource in candidate_resources: for report in resource.extra_data.get("symbols_reachability", []): advisory_uids = report.get("advisory_uids", []) - is_reachable = report.get("is_reachable") + is_reachable = ( + report.get("is_reachable") or ReachabilityStatus.NOT_REACHABLE.value + ) patch = report.get("patch", {}) for adv_uid in advisory_uids: diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 7afdd93dad..6e7b6f92df 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -246,7 +246,7 @@ def test_generate_advisory_reachability_report(self): "vcs_url": "https://example.com", "commit_hash": "def456", }, - "is_reachable": None, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, "tool_details": [], "vulnerable_symbols": [], "fixed_symbols": [], @@ -257,7 +257,7 @@ def test_generate_advisory_reachability_report(self): "vcs_url": "https://example.com", "commit_hash": "abc123", }, - "is_reachable": None, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, "tool_details": [], "vulnerable_symbols": ["vuln_sym1"], "fixed_symbols": ["fixed_sym1"], @@ -274,7 +274,7 @@ def test_generate_advisory_reachability_report(self): "vcs_url": "https://example2.com", "commit_hash": "46a4asf", }, - "is_reachable": None, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, "tool_details": [], "vulnerable_symbols": [], "fixed_symbols": [], @@ -285,7 +285,7 @@ def test_generate_advisory_reachability_report(self): "vcs_url": "https://example.com", "commit_hash": "abc123", }, - "is_reachable": None, + "is_reachable": ReachabilityStatus.NOT_REACHABLE.value, "tool_details": [], "vulnerable_symbols": ["vuln_sym1"], "fixed_symbols": ["fixed_sym1"], From 07ed82f1601af0ac6e6ff5a2fcd49858441637a4 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 27 Aug 2026 06:14:31 +0300 Subject: [PATCH 43/50] Fix a bug related to vulnerability dependency reachability pipeline Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 33 +++-- .../tests/pipes/test_symbols_reachability.py | 124 +++++++++++++++++- 2 files changed, 147 insertions(+), 10 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 9d3cf3f9ad..5bd47ff746 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -429,12 +429,9 @@ def build_index(self): callers_of = {} # callee_name -> set of caller_qualified_names for node, _ in lang_query.get_functions(tree.root_node): - qualified_name = self.process_node( + self.process_node( node, extractor, definitions_index, definitions, fingerprints ) - if qualified_name: - for _, callee_name in extractor.extract_calls(node): - callers_of.setdefault(callee_name, set()).add(qualified_name) for node, _ in lang_query.get_classes(tree.root_node): self.process_node( @@ -446,6 +443,22 @@ def build_index(self): node, extractor, definitions_index, definitions, fingerprints ) + for receiver_node, callee_node in lang_query.get_calls(tree.root_node): + callee_name = callee_node.text.decode("utf-8", errors="replace") + if not callee_name: + continue + + caller_name = None + curr = callee_node + while curr is not None: + def_info = definitions_index.get(curr.id) + if def_info and def_info.get("qualified_name"): + caller_name = def_info["qualified_name"] + break + curr = curr.parent + + callers_of.setdefault(callee_name, set()).add(caller_name) + return { "definitions": definitions, "fingerprints": fingerprints, @@ -582,7 +595,7 @@ def match(self, patch_symbols_metadata): entry["is_exact"] = entry["is_exact"] or is_exact entry["is_called"] = entry["is_called"] or called if called: - entry["reachable_from"] = sorted(callers) + entry["reachable_from"] = sorted([c for c in callers if c is not None]) return matched @@ -709,12 +722,14 @@ def collect_and_match_resources( if not resource_index: continue - patch_symbols = patch_symbols_by_language.get(resource.programming_language) - if not patch_symbols: + lang_patch_symbols = patch_symbols_by_language.get( + resource.programming_language + ) + if not lang_patch_symbols: continue - vulnerable_symbols = patch_symbols.get("vulnerable", {}) - fixed_symbols = patch_symbols.get("fixed", {}) + vulnerable_symbols = lang_patch_symbols.get("vulnerable", {}) + fixed_symbols = lang_patch_symbols.get("fixed", {}) matcher = ResourcePatchMatcher(resource_index=resource_index) vuln_details = matcher.match(vulnerable_symbols) diff --git a/scanpipe/tests/pipes/test_symbols_reachability.py b/scanpipe/tests/pipes/test_symbols_reachability.py index 6e7b6f92df..bc9b8a866d 100644 --- a/scanpipe/tests/pipes/test_symbols_reachability.py +++ b/scanpipe/tests/pipes/test_symbols_reachability.py @@ -403,7 +403,7 @@ def test_python_get_symbol_reachability_results( "is_defined": True, "is_imported": False, "symbol_name": "serve_report.build_file_path", - "reachable_from": ["serve_report"], + "reachable_from": ["serve_report.target_path"], }, { "is_exact": True, @@ -439,6 +439,72 @@ def test_python_get_symbol_reachability_results( expected_results, ) + @patch("scanpipe.pipes.reachability.Repo") + @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") + @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) + def test_dependency_simple_reachability( + self, + mock_package_vulnerabilities, + mock_collect_symbols, + mock_repo, + ): + """ + Test the reachability pipeline + for a vulnerability dependency. + """ + file_path = "app.py" + + app_text = ( + "from aiohttp.http_parser import HttpRequestParser\n\n" + "HttpRequestParser.parse_message()\n" + ) + + vuln_text = ( + "class HttpRequestParser:\n" + " def parse_message(self):\n" + " return eval('1')\n" + ) + + fixed_text = ( + "class HttpRequestParser:\n" + " def parse_message(self):\n" + " return int('1')\n" + ) + + expected_results = [ + { + "patch": { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + }, + "is_reachable": "yes", + "tool_details": [ + { + "is_exact": False, + "is_called": True, + "is_defined": False, + "is_imported": True, + "symbol_name": "HttpRequestParser.parse_message", + "reachable_from": [], + } + ], + "advisory_uids": [], + "fixed_symbols": ["HttpRequestParser.parse_message"], + "vulnerable_symbols": ["HttpRequestParser.parse_message"], + } + ] + + self._run_reachability_pipeline( + mock_package_vulnerabilities, + mock_collect_symbols, + mock_repo, + file_path, + app_text, + vuln_text, + fixed_text, + expected_results, + ) + @patch("scanpipe.pipes.reachability.Repo") @patch("scanpipe.pipes.reachability.PatchAnalyzer.collect_patch_symbols") @patch.object(Project, "package_vulnerabilities", new_callable=PropertyMock) @@ -1045,10 +1111,19 @@ def deep_func(): def relative_func(): return eval("8") +def module_func(): + return eval("9") + +def unused_func(): + return eval("10") + class MyClass: def class_method(self): return eval("4") + def target_method(self): + return eval("11") + class InnerClass: def deep_method(self): return eval("5") @@ -1073,10 +1148,19 @@ def deep_func(): def relative_func(): return int("8") +def module_func(): + return int("9") + +def unused_func(): + return int("10") + class MyClass: def class_method(self): return int("4") + def target_method(self): + return int("11") + class InnerClass: def deep_method(self): return int("5") @@ -1093,6 +1177,11 @@ def deep_method(self): from my_module import ( multiline_func, ) +from my_module import module_func as mf +from my_module import unused_func + +# 1. Module-level call via alias +mf() def execute(): direct_func() @@ -1104,9 +1193,22 @@ def execute(): c = C() c.class_method() + c.target_method() inner = C.InnerClass() inner.deep_method() + +def execute_nested(): + def inner_helper(): + # 2. Call inside a nested function via alias + mf() + inner_helper() + +def wildcard_caller(): + # 3. Call via wildcard import + module_func() + +# unused_func is imported but never called """.strip() file_path = "my_module.py" @@ -1189,6 +1291,26 @@ def execute(): self.assertTrue(matched["relative_func"]["is_called"]) self.assertEqual(matched["relative_func"]["reachable_from"], ["execute"]) + self.assertIn("MyClass.target_method", matched) + self.assertTrue(matched["MyClass.target_method"]["is_imported"]) + self.assertTrue(matched["MyClass.target_method"]["is_called"]) + self.assertEqual( + matched["MyClass.target_method"]["reachable_from"], ["execute"] + ) + + self.assertIn("module_func", matched) + self.assertTrue(matched["module_func"]["is_imported"]) + self.assertTrue(matched["module_func"]["is_called"]) + self.assertEqual( + matched["module_func"]["reachable_from"], + ["execute_nested.inner_helper", "wildcard_caller"], + ) + + self.assertIn("unused_func", matched) + self.assertTrue(matched["unused_func"]["is_imported"]) + self.assertFalse(matched["unused_func"]["is_called"]) + self.assertEqual(matched["unused_func"]["reachable_from"], []) + def test_resource_patch_matcher_java(self): """ Test matching From e65b8e2a4c1a3aadf3df6e95776ceaa786b97a7e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 28 Aug 2026 05:31:20 +0300 Subject: [PATCH 44/50] Update function docs Add resource filter based on language we support Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 33 +++++++++---------- scanpipe/pipes/reachability.py | 29 ++++++++-------- 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 06c2d6318f..17ac91ab93 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -22,23 +22,24 @@ from scanpipe.pipelines import Pipeline from scanpipe.pipes import reachability +from scanpipe.pipes.symbols import TS_QUERIES class SymbolReachability(Pipeline): """ - Patch reachability analysis for given vulnerability patches. + Determine the reachability of vulnerabilities identified in the project. - For every patch we clone the git repository and extract - the vulnerable and fixed symbols from the patch commit. + Note: You must run `find_vulnerabilities` pipeline before running this pipeline. - These symbols are then matched against the project's codebase resources to - determine if the vulnerable code is actually present and reachable. + For every patch the git repository is cloned and extract the vulnerable and fixed + symbols from the patch commit. These symbols are then matched against + the project's codebase resources to determine if the vulnerable code + is actually present and reachable. - The analysis checks if vulnerable symbols are defined, imported, called, or - exactly match a fingerprint within the project files. The results, including - tool_details and a reachability status (REACHABLE, UNKNOWN, or - NOT_REACHABLE), are stored in the `extra_data` of the matching resources - under the `symbols_reachability` key. + The analysis checks if vulnerable symbols are defined, imported, called, + or exactly match a code within the project files. The results, including + tool_details and a reachability status (yes, unknown, or no), are stored + in the `extra_data` of the matching resources under the `symbols_reachability` key. Finally, a summary report is generated for each vulnerability advisory and saved as a JSON output file. @@ -66,24 +67,22 @@ def get_vulnerabilities_patches(self): ) def collect_resource_index(self): - """Collect resources symbols for each resource exactly once.""" + """Collect resources symbols for each resource""" self.candidate_resources = self.project.codebaseresources.files().filter( - is_binary=False, is_archive=False, is_media=False + is_binary=False, is_archive=False, is_media=False, + programming_language__in=TS_QUERIES.keys() ) self.resource_indexes = reachability.collect_resource_index( candidate_resources=self.candidate_resources ) def collect_patch_symbols(self): - """ - Collect patch symbols for all related commits, - then remove the local clone before moving on - """ + """Collect patch symbols for all related commits.""" self.patch_symbols = reachability.collect_patch_symbols(patches=self.patches) def collect_and_match_resources(self): """Match resource symbols against patch symbols.""" - reachability.collect_and_match_resources( + reachability.match_patches_to_resources( patches=self.patches, patch_symbols=self.patch_symbols, resource_indexes=self.resource_indexes, diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 5bd47ff746..efd5a15f3b 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -29,7 +29,7 @@ from git import Repo from git.diff import NULL_TREE -from scancode.api import get_file_info +from typecode import get_type from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor @@ -55,10 +55,7 @@ def normalize_text(content): def detect_language_with_scancode(file_path, content): - """ - Detect the programming language of the text - content using get_file_info function - """ + """Detect the programming language of the text""" content = normalize_text(content) if not content: @@ -67,11 +64,11 @@ def detect_language_with_scancode(file_path, content): tmp_dir = tempfile.mkdtemp(prefix="patch-lang-") try: - target = Path(tmp_dir) / Path(file_path).name - target.write_text(content, encoding="utf-8", errors="replace") + location = Path(tmp_dir) / Path(file_path).name + location.write_text(content, encoding="utf-8", errors="replace") - info = get_file_info(location=str(target)) or {} - return info.get("programming_language") or None + info = get_type(location) + return info.programming_language or None finally: shutil.rmtree(tmp_dir, ignore_errors=True) @@ -652,7 +649,7 @@ def get_vulnerabilities_patches(package_vulnerabilities, dependency_vulnerabilit def collect_resource_index(candidate_resources): - """Collect resources symbols for each resource exactly once.""" + """Collect resources symbols for each resource""" resource_indexes = {} for resource in candidate_resources: resource_language = resource.programming_language @@ -674,8 +671,8 @@ def collect_resource_index(candidate_resources): def collect_patch_symbols(patches): """ - For each unique repo: clone it once, collect patch symbols for all - related commits, then remove the local clone before moving on + For each unique repo clone it once, + collect patch symbols for all related commits """ patch_symbols = {} patches_by_repo = {} @@ -704,7 +701,7 @@ def collect_patch_symbols(patches): return patch_symbols -def collect_and_match_resources( +def match_patches_to_resources( patches, patch_symbols, candidate_resources, resource_indexes ): """Match resource symbols against patch symbols.""" @@ -760,18 +757,18 @@ def collect_and_match_resources( def generate_advisory_reachability_report(project, patches, candidate_resources): """ - Generate reachability report keyed by advisory_uid. + Generate a reachability report summarizing status by advisory. Each advisory contains its overall reachability status and the reachability results collected from all resources and associated patches. The overall reachability status is determined using the following - priority order: REACHABLE > UNKNOWN > NOT_REACHABLE + priority order: REACHABLE: "yes" > UNKNOWN: "unknown" > NOT_REACHABLE: "no" This means that an advisory is considered reachable if it is reachable through at least one resource or patch. If no reachable result exists, - but at least one result is unknown, the advisory status is UNKNOWN. + but at least one result is UNKNOWN, the advisory status is UNKNOWN. Otherwise, it is NOT_REACHABLE. """ status_priority = { From 40d80ca1d57576d85fa88a5c674da64178f76a9e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 28 Aug 2026 05:52:22 +0300 Subject: [PATCH 45/50] Fix a typo Signed-off-by: ziad hany --- scanpipe/pipelines/analyze_symbols_reachability.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 17ac91ab93..9ecf011853 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -90,7 +90,7 @@ def collect_and_match_resources(self): ) def generate_advisory_reachability_report(self): - """Generate reachability report keyed by advisory_uid.""" + """Generate a reachability report summarizing status by advisory.""" reachability.generate_advisory_reachability_report( project=self.project, patches=self.patches, From 83200b3c1d582bb48390a1b3bf72d5dab48066d5 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 28 Aug 2026 06:03:05 +0300 Subject: [PATCH 46/50] Fix format error Signed-off-by: ziad hany --- scanpipe/pipelines/analyze_symbols_reachability.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 9ecf011853..b59213b74d 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -69,8 +69,10 @@ def get_vulnerabilities_patches(self): def collect_resource_index(self): """Collect resources symbols for each resource""" self.candidate_resources = self.project.codebaseresources.files().filter( - is_binary=False, is_archive=False, is_media=False, - programming_language__in=TS_QUERIES.keys() + is_binary=False, + is_archive=False, + is_media=False, + programming_language__in=TS_QUERIES.keys(), ) self.resource_indexes = reachability.collect_resource_index( candidate_resources=self.candidate_resources From bdc627e0e32dc11a291bafb1fadfa750db926746 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Fri, 28 Aug 2026 18:10:39 +0300 Subject: [PATCH 47/50] Add a LoopProgress to the most time-consuming functions Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 7 +++++-- scanpipe/pipes/reachability.py | 19 +++++++++++++------ 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index b59213b74d..6084612316 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -75,12 +75,14 @@ def collect_resource_index(self): programming_language__in=TS_QUERIES.keys(), ) self.resource_indexes = reachability.collect_resource_index( - candidate_resources=self.candidate_resources + candidate_resources=self.candidate_resources, logger=self.log ) def collect_patch_symbols(self): """Collect patch symbols for all related commits.""" - self.patch_symbols = reachability.collect_patch_symbols(patches=self.patches) + self.patch_symbols = reachability.collect_patch_symbols( + patches=self.patches, logger=self.log + ) def collect_and_match_resources(self): """Match resource symbols against patch symbols.""" @@ -89,6 +91,7 @@ def collect_and_match_resources(self): patch_symbols=self.patch_symbols, resource_indexes=self.resource_indexes, candidate_resources=self.candidate_resources, + logger=self.log, ) def generate_advisory_reachability_report(self): diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index efd5a15f3b..844324bb65 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -31,6 +31,7 @@ from git.diff import NULL_TREE from typecode import get_type +from aboutcode.pipeline import LoopProgress from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint @@ -648,10 +649,12 @@ def get_vulnerabilities_patches(package_vulnerabilities, dependency_vulnerabilit return list(patches.values()) -def collect_resource_index(candidate_resources): +def collect_resource_index(candidate_resources, logger=None): """Collect resources symbols for each resource""" resource_indexes = {} - for resource in candidate_resources: + resources_count = len(candidate_resources) + progress = LoopProgress(resources_count, logger) + for resource in progress.iter(candidate_resources): resource_language = resource.programming_language file_content = normalize_text(resource.file_content) @@ -669,7 +672,7 @@ def collect_resource_index(candidate_resources): return resource_indexes -def collect_patch_symbols(patches): +def collect_patch_symbols(patches, logger=None): """ For each unique repo clone it once, collect patch symbols for all related commits @@ -680,7 +683,9 @@ def collect_patch_symbols(patches): vcs_url = patch.get("vcs_url") patches_by_repo.setdefault(vcs_url, []).append(patch) - for vcs_url, repo_patches in patches_by_repo.items(): + repo_count = len(patches_by_repo) + repo_progress = LoopProgress(repo_count, logger) + for vcs_url, repo_patches in repo_progress.iter(patches_by_repo.items()): with tempfile.TemporaryDirectory(prefix="symbol-reachability-") as repo_path: try: repo = Repo.clone_from(vcs_url, repo_path) @@ -702,10 +707,12 @@ def collect_patch_symbols(patches): def match_patches_to_resources( - patches, patch_symbols, candidate_resources, resource_indexes + patches, patch_symbols, candidate_resources, resource_indexes, logger=None ): """Match resource symbols against patch symbols.""" - for patch in patches: + patches_count = len(patches) + patch_progress = LoopProgress(patches_count, logger=logger) + for patch in patch_progress.iter(patches): vcs_url = patch.get("vcs_url") commit_hash = patch.get("commit_hash") advisory_uids = patch.get("advisory_uids", []) From 353548648a182731decf8d168eaf4c82df7a89e8 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 25 Aug 2026 02:41:59 +0300 Subject: [PATCH 48/50] Expose reachability in API for package and dependency Signed-off-by: ziad hany --- .../pipelines/analyze_symbols_reachability.py | 11 +++ scanpipe/pipes/reachability.py | 67 ++++++++++++++- scanpipe/tests/test_api.py | 83 +++++++++++++++++++ 3 files changed, 157 insertions(+), 4 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 6084612316..2c747c405b 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -57,6 +57,7 @@ def steps(cls): cls.collect_patch_symbols, cls.collect_and_match_resources, cls.generate_advisory_reachability_report, + cls.apply_reachability_to_packages_and_dependencies, ) def get_vulnerabilities_patches(self): @@ -101,3 +102,13 @@ def generate_advisory_reachability_report(self): patches=self.patches, candidate_resources=self.candidate_resources, ) + + def apply_reachability_to_packages_and_dependencies(self): + """ + Save reachability results by updating DiscoveredPackage and + DiscoveredDependency records with the computed reachability data + in their affected_by_vulnerabilities JSON field. + """ + reachability.apply_reachability_to_packages_and_dependencies( + project=self.project, advisory_report=self.advisory_map + ) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 844324bb65..3e95081f17 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -32,6 +32,8 @@ from typecode import get_type from aboutcode.pipeline import LoopProgress +from scanpipe.models import DiscoveredDependency +from scanpipe.models import DiscoveredPackage from scanpipe.pipes.symbols import TS_QUERIES from scanpipe.pipes.symbols import SymbolExtractor from scanpipe.pipes.symbols import create_sha256_fingerprint @@ -784,7 +786,7 @@ def generate_advisory_reachability_report(project, patches, candidate_resources) ReachabilityStatus.NOT_REACHABLE.value: 1, } - advisory_reachability_report = { + advisories_reachability_report = { "purl": project.purl, "advisories": [], } @@ -799,7 +801,7 @@ def generate_advisory_reachability_report(project, patches, candidate_resources) "details": [], } advisory_map[adv_uid] = adv_data - advisory_reachability_report["advisories"].append(adv_data) + advisories_reachability_report["advisories"].append(adv_data) for resource in candidate_resources: for report in resource.extra_data.get("symbols_reachability", []): @@ -817,7 +819,7 @@ def generate_advisory_reachability_report(project, patches, candidate_resources) "details": [], } advisory_map[adv_uid] = adv_data - advisory_reachability_report["advisories"].append(adv_data) + advisories_reachability_report["advisories"].append(adv_data) tool_details = { "resource_path": resource.path, @@ -839,4 +841,61 @@ def generate_advisory_reachability_report(project, patches, candidate_resources) reachability_output_path = project.get_output_file_path("reachability", "json") with open(reachability_output_path, "w") as f: - json.dump(advisory_reachability_report, f, indent=2) + json.dump(advisories_reachability_report, f, indent=2) + + return advisories_reachability_report + + +def inject_reachability_data(vulns, advisory_map): + """ + Inject reachability data into a list of vulnerabilities. + Returns True if any vulnerability was updated, False otherwise. + """ + updated = False + for vuln in vulns: + adv_uid = vuln.get("advisory_uid") + if adv_uid in advisory_map: + adv_data = advisory_map[adv_uid] + vuln["is_reachable"] = adv_data.get("is_reachable", "unknown") + vuln["reachability_analysis"] = adv_data.get("details", []) + updated = True + + return updated + + +def apply_reachability_to_packages_and_dependencies(project, advisory_report): + """ + Update DiscoveredPackage and DiscoveredDependency records by injecting the + computed reachability data into their affected_by_vulnerabilities JSON field. + """ + advisories_list = advisory_report.get("advisories", []) + if not advisories_list: + return + + advisory_map = {adv["advisory_uid"]: adv for adv in advisories_list} + + unsaved_packages = [] + for package in project.discoveredpackages.all(): + vulns = package.affected_by_vulnerabilities or [] + if inject_reachability_data(vulns, advisory_map): + unsaved_packages.append(package) + + if unsaved_packages: + DiscoveredPackage.objects.bulk_update( + objs=unsaved_packages, + fields=["affected_by_vulnerabilities"], + batch_size=10, + ) + + unsaved_deps = [] + for dep in project.discovereddependencies.all(): + vulns = dep.affected_by_vulnerabilities or [] + if inject_reachability_data(vulns, advisory_map): + unsaved_deps.append(dep) + + if unsaved_deps: + DiscoveredDependency.objects.bulk_update( + objs=unsaved_deps, + fields=["affected_by_vulnerabilities"], + batch_size=10, + ) diff --git a/scanpipe/tests/test_api.py b/scanpipe/tests/test_api.py index 3fb58b3f7f..90719208d9 100644 --- a/scanpipe/tests/test_api.py +++ b/scanpipe/tests/test_api.py @@ -56,6 +56,7 @@ from scanpipe.models import WebhookSubscription from scanpipe.pipes.input import copy_input from scanpipe.pipes.output import JSONResultsGenerator +from scanpipe.pipes.reachability import apply_reachability_to_packages_and_dependencies from scanpipe.tests import dependency_data1 from scanpipe.tests import filter_warnings from scanpipe.tests import make_message @@ -1374,3 +1375,85 @@ def test_scanpipe_api_serializer_get_serializer_fields(self): with self.assertRaises(LookupError): get_serializer_fields(None) + + def test_scanpipe_api_project_action_package_with_reachability(self): + self.discovered_package1.affected_by_vulnerabilities = [ + { + "advisory_id": "PYSEC-2026-1", + "advisory_uid": "pypa/scancode/PYSEC-2026-1", + "summary": "summary 1", + "risk_score": 1, + }, + { + "advisory_id": "PYSEC-2026-2", + "advisory_uid": "pypa/scancode/PYSEC-2026-2", + "summary": "summary 2", + "risk_score": 2, + }, + { + "advisory_id": "PYSEC-2026-3", + "advisory_uid": "pypa/scancode/PYSEC-2026-3", + "summary": "summary 3", + "risk_score": 3, + }, + ] + self.discovered_package1.save() + advisory_map = { + "purl": "pkg:pypi/daglib@0.3.2", + "advisories": [ + { + "advisory_uid": "pypa/scancode/PYSEC-2026-1", + "is_reachable": "unknown", + "details": [ + { + "resource_path": "scancode/session.py", + "is_reachable": "unknown", + "vulnerable_symbols": ["SqliteAccountInfo"], + } + ], + }, + { + "advisory_uid": "pypa/scancode/PYSEC-2026-2", + "is_reachable": "yes", + "details": [ + { + "resource_path": "b2sdk/session.py", + "is_reachable": "yes", + "vulnerable_symbols": ["SqliteAccountInfo"], + } + ], + }, + ], + } + + apply_reachability_to_packages_and_dependencies(self.project1, advisory_map) + url = reverse("project-packages", args=[self.project1.uuid]) + response = self.csrf_client.get(url) + + self.assertEqual(status.HTTP_200_OK, response.status_code) + self.assertEqual(1, response.data["count"]) + + pkg_response = response.data["results"][0] + vulns = pkg_response["affected_by_vulnerabilities"] + + self.assertEqual(3, len(vulns)) + + self.assertEqual("pypa/scancode/PYSEC-2026-1", vulns[0]["advisory_uid"]) + self.assertEqual("unknown", vulns[0]["is_reachable"]) + self.assertIn("reachability_analysis", vulns[0]) + self.assertEqual(1, len(vulns[0]["reachability_analysis"])) + self.assertEqual( + "scancode/session.py", vulns[0]["reachability_analysis"][0]["resource_path"] + ) + + self.assertEqual("pypa/scancode/PYSEC-2026-2", vulns[1]["advisory_uid"]) + self.assertEqual("yes", vulns[1]["is_reachable"]) + self.assertIn("reachability_analysis", vulns[1]) + self.assertEqual(1, len(vulns[1]["reachability_analysis"])) + self.assertEqual( + "b2sdk/session.py", vulns[1]["reachability_analysis"][0]["resource_path"] + ) + + self.assertEqual("pypa/scancode/PYSEC-2026-3", vulns[2]["advisory_uid"]) + self.assertNotIn("is_reachable", vulns[2]) + self.assertNotIn("reachability_analysis", vulns[2]) From fc917e796f720040c41d7d0a987b852cd47204a2 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 25 Aug 2026 14:39:49 +0300 Subject: [PATCH 49/50] Simplify the apply_reachability_to_packages_and_dependencies function Signed-off-by: ziad hany --- scanpipe/pipes/reachability.py | 51 +++++++++++++++------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index 3e95081f17..ee0dc5993a 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -868,34 +868,27 @@ def apply_reachability_to_packages_and_dependencies(project, advisory_report): Update DiscoveredPackage and DiscoveredDependency records by injecting the computed reachability data into their affected_by_vulnerabilities JSON field. """ - advisories_list = advisory_report.get("advisories", []) - if not advisories_list: + advisories = advisory_report.get("advisories", []) + if not advisories: return - advisory_map = {adv["advisory_uid"]: adv for adv in advisories_list} - - unsaved_packages = [] - for package in project.discoveredpackages.all(): - vulns = package.affected_by_vulnerabilities or [] - if inject_reachability_data(vulns, advisory_map): - unsaved_packages.append(package) - - if unsaved_packages: - DiscoveredPackage.objects.bulk_update( - objs=unsaved_packages, - fields=["affected_by_vulnerabilities"], - batch_size=10, - ) - - unsaved_deps = [] - for dep in project.discovereddependencies.all(): - vulns = dep.affected_by_vulnerabilities or [] - if inject_reachability_data(vulns, advisory_map): - unsaved_deps.append(dep) - - if unsaved_deps: - DiscoveredDependency.objects.bulk_update( - objs=unsaved_deps, - fields=["affected_by_vulnerabilities"], - batch_size=10, - ) + advisory_map = {adv["advisory_uid"]: adv for adv in advisories} + targets = ( + (project.discoveredpackages.all(), DiscoveredPackage), + (project.discovereddependencies.all(), DiscoveredDependency), + ) + + for queryset, model in targets: + unsaved = [ + item + for item in queryset + if inject_reachability_data( + item.affected_by_vulnerabilities or [], advisory_map + ) + ] + if unsaved: + model.objects.bulk_update( + objs=unsaved, + fields=["affected_by_vulnerabilities"], + batch_size=10, + ) From 96a827abd3d93973b2327edfe87e0ff983d4bea0 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Sat, 29 Aug 2026 00:43:39 +0300 Subject: [PATCH 50/50] Resolve merge conflict Signed-off-by: ziad hany --- scanpipe/pipelines/analyze_symbols_reachability.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 2c747c405b..4eaf4116e1 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -97,10 +97,12 @@ def collect_and_match_resources(self): def generate_advisory_reachability_report(self): """Generate a reachability report summarizing status by advisory.""" - reachability.generate_advisory_reachability_report( - project=self.project, - patches=self.patches, - candidate_resources=self.candidate_resources, + self.advisories_reachability_report = ( + reachability.generate_advisory_reachability_report( + project=self.project, + patches=self.patches, + candidate_resources=self.candidate_resources, + ) ) def apply_reachability_to_packages_and_dependencies(self): @@ -110,5 +112,5 @@ def apply_reachability_to_packages_and_dependencies(self): in their affected_by_vulnerabilities JSON field. """ reachability.apply_reachability_to_packages_and_dependencies( - project=self.project, advisory_report=self.advisory_map + project=self.project, advisory_report=self.advisories_reachability_report )