diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 48f3ec6..e064cd6 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -16,16 +16,16 @@ jobs: contents: read packages: write steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: trailofbits/setup-codeql@615e3864087261d42cce229e3eec419ab9b22c36 # main + - uses: trailofbits/setup-codeql@b8013ac1f220d4c372aca61c2baa643e07497165 # main with: - version: '2.25.1' + version: '2.27.4' platform: 'linux64' - checksum: '4f070e6cc7009e75aec307ed109c2fcf0501e579c20a31080b893e31209523d5' + checksum: '393bf92b66596141ec93baf9b483f4a350f3749916c922fed0dd63c67706047d' - name: Log in to the Container registry - uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 + uses: docker/login-action@8aa5831fb8d48d8f76594c7ddde61ae58019f550 # v4.8.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e3b766a..7012ab3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,15 +12,15 @@ jobs: main: runs-on: ubuntu-latest steps: - - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: trailofbits/setup-codeql@615e3864087261d42cce229e3eec419ab9b22c36 # main + - uses: trailofbits/setup-codeql@b8013ac1f220d4c372aca61c2baa643e07497165 # main with: - version: '2.25.1' + version: '2.27.4' platform: 'linux64' - checksum: '4f070e6cc7009e75aec307ed109c2fcf0501e579c20a31080b893e31209523d5' - - uses: astral-sh/setup-uv@cec208311dfd045dd5311c1add060b2062131d57 # v8.0.0 + checksum: '393bf92b66596141ec93baf9b483f4a350f3749916c922fed0dd63c67706047d' + - uses: astral-sh/setup-uv@68559e99cc807c1eb7e9238fd43a15bba460eec2 # v10.3.3 - run: make format-check - run: make test - name: Verify doc/QUERIES.md is up to date diff --git a/scripts/queries_table_generator.py b/scripts/queries_table_generator.py index b652cba..3ab3cbc 100644 --- a/scripts/queries_table_generator.py +++ b/scripts/queries_table_generator.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python - """Generate markdown tables for all queries. Queries are listed from all Qlpacks' "full" suites, grouped by the custom @@ -12,7 +10,6 @@ from dataclasses import dataclass from functools import total_ordering from pathlib import Path -from typing import Optional import yaml @@ -27,9 +24,9 @@ class QueryMetadata: kind: str tags: str group: str - problem_severity: Optional[str] - precision: Optional[str] - security_severity: Optional[str] + problem_severity: str | None + precision: str | None + security_severity: str | None @total_ordering @@ -49,12 +46,12 @@ def __init__(self, path, doc_lang_dir: str, metadata: QueryMetadata): self.id: str = metadata.id self.description: str = metadata.description self.kind: str = metadata.kind - self.tags: list[str] = sorted(metadata.tags.split(' ')) + self.tags: list[str] = sorted(metadata.tags.split(" ")) self.group: str = metadata.group - self.problem_severity: Optional[str] = metadata.problem_severity - self.precision: Optional[str] = metadata.precision - self.security_severity: Optional[float] = ( + self.problem_severity: str | None = metadata.problem_severity + self.precision: str | None = metadata.precision + self.security_severity: float | None = ( float(metadata.security_severity) if metadata.security_severity is not None else None @@ -71,8 +68,8 @@ def __gt__(self, other: "QlQuery"): return self.group < other.group # Experimental queries sort after non-experimental ones. - self_exp = 'experimental' in self.tags - other_exp = 'experimental' in other.tags + self_exp = "experimental" in self.tags + other_exp = "experimental" in other.tags if self_exp and not other_exp: return False if other_exp and not self_exp: @@ -82,10 +79,10 @@ def __gt__(self, other: "QlQuery"): def md_table_line(self): qhelp_markdown_path = ( - Path('..') / self.doc_lang_dir / 'src' / 'docs' / self.rel_path - ).with_suffix('.md') + Path("..") / self.doc_lang_dir / "src" / "docs" / self.rel_path + ).with_suffix(".md") cells = [ - f'[{self.name}]({qhelp_markdown_path})', + f"[{self.name}]({qhelp_markdown_path})", f"{self.description}", f"{self.problem_severity}", f"{self.precision}", @@ -97,7 +94,7 @@ def md_table_line(self): @staticmethod def parse_from_file(path: Path, doc_lang_dir: str) -> "QlQuery": raw_metadata = subprocess.check_output( - ['codeql', 'resolve', 'metadata', str(path)] + ["codeql", "resolve", "metadata", str(path)] ).decode("utf-8") parsed = json.loads(raw_metadata) @@ -133,7 +130,7 @@ def __init__(self, path: Path, queries: list[QlQuery], doc_lang_dir: str): def md_tables(self) -> str: tables = "" - for group, query_list in self.queries.items(): + for group, query_list in sorted(self.queries.items()): tables += f"#### {group.capitalize()}\n\n" table = "" @@ -149,13 +146,19 @@ def md_tables(self) -> str: @staticmethod def get_qls_from_path(qls_path: Path, doc_lang_dir: str) -> "Qls": - queries_paths = subprocess.check_output([ - 'codeql', - 'resolve', - 'queries', - '--search-path=.', - str(qls_path), - ]).decode("utf-8").split("\n") + queries_paths = ( + subprocess.check_output( + [ + "codeql", + "resolve", + "queries", + "--search-path=.", + str(qls_path), + ] + ) + .decode("utf-8") + .split("\n") + ) queries = [] for path in queries_paths: @@ -170,10 +173,10 @@ def get_qls_from_path(qls_path: Path, doc_lang_dir: str) -> "Qls": def display_name(extractor: str) -> str: """Return the human-readable language heading for an extractor name.""" - if extractor == 'cpp': - return 'C and C++' - if extractor == 'java-kotlin': - return 'Java and Kotlin' + if extractor == "cpp": + return "C and C++" + if extractor == "java-kotlin": + return "Java and Kotlin" return extractor.capitalize() @@ -181,17 +184,23 @@ def main(): # get ToB qlpacks that are not tests nor libraries tob_qlpacks = {} qlpack_paths = json.loads( - subprocess.check_output([ - 'codeql', - 'resolve', - 'qlpacks', - '--search-path=.', - '--format=json', - ]).decode("utf-8") + subprocess.check_output( + [ + "codeql", + "resolve", + "qlpacks", + "--search-path=.", + "--format=json", + ] + ).decode("utf-8") ) for k, v in qlpack_paths.items(): - if k.startswith('trailofbits/') and not k.endswith('-tests') and not k.endswith('-all'): + if ( + k.startswith("trailofbits/") + and not k.endswith("-tests") + and not k.endswith("-all") + ): tob_qlpacks[k] = Path(v[0]) sys.stdout.write( @@ -200,22 +209,20 @@ def main(): # for every pack finds the "full" suite for qlpack_name, qlpack_path in tob_qlpacks.items(): - with open(qlpack_path / 'qlpack.yml') as f: + with open(qlpack_path / "qlpack.yml") as f: qlpack = yaml.safe_load(f) # skip libraries - if qlpack.get('library') is True: + if qlpack.get("library") is True: continue # skip tests - if qlpack.get('version') is None: + if qlpack.get("version") is None: continue - suites_dir = qlpack.get('suites', 'codeql-suites') - suites = list((qlpack_path / suites_dir).glob('*-full.qls')) + suites_dir = qlpack.get("suites", "codeql-suites") + suites = list((qlpack_path / suites_dir).glob("*-full.qls")) if len(suites) > 1: - sys.exit( - f'Error: Found more than 1 "full" suite for qlpack {qlpack_name}' - ) + sys.exit(f'Error: Found more than 1 "full" suite for qlpack {qlpack_name}') if len(suites) == 0: sys.exit(f'Error: No "full" suite for qlpack {qlpack_name}') @@ -229,7 +236,7 @@ def main(): doc_lang_dir = qlpack_path.parent.name suite = Qls.get_qls_from_path(suites[0], doc_lang_dir) - sys.stdout.write(f'### {display_name(extractor)}\n\n') + sys.stdout.write(f"### {display_name(extractor)}\n\n") sys.stdout.write(suite.md_tables())