From 69d847897e0d2445e1d859169f14750f4c8fe1a8 Mon Sep 17 00:00:00 2001 From: Louis1125 Date: Sat, 22 Aug 2026 22:33:12 +0800 Subject: [PATCH 1/3] perf(ci): parallelize plugin validation and add sha256 workspace caching --- .github/scripts/utils/cache_manager.py | 55 ++++++++++++++++++++++++ .github/scripts/validate.py | 58 ++++++++++++++++++++++++++ .github/workflows/validate.yml | 30 +++++++++++++ .gitignore | 3 ++ README.md | 7 ++++ package-lock.json | 6 +++ 6 files changed, 159 insertions(+) create mode 100644 .github/scripts/utils/cache_manager.py create mode 100644 .github/scripts/validate.py create mode 100644 .github/workflows/validate.yml create mode 100644 package-lock.json diff --git a/.github/scripts/utils/cache_manager.py b/.github/scripts/utils/cache_manager.py new file mode 100644 index 00000000..10fc1c4b --- /dev/null +++ b/.github/scripts/utils/cache_manager.py @@ -0,0 +1,55 @@ +import os +import json +import hashlib + +CACHE_FILE = os.path.join(os.getcwd(), ".cache", "plugin_validation.json") + +class ValidationCache: + def __init__(self): + self.cache = self._load_cache() + + def _load_cache(self): + if os.path.exists(CACHE_FILE): + try: + with open(CACHE_FILE, "r", encoding="utf-8") as f: + return json.load(f) + except Exception: + pass + return {} + + def get_hash(self, target_path): + hasher = hashlib.sha256() + + if os.path.isfile(target_path): + with open(target_path, "rb") as f: + while chunk := f.read(8192): + hasher.update(chunk) + elif os.path.isdir(target_path): + for root, dirs, files in os.walk(target_path): + # Ignore .git and .cache folders + dirs[:] = [d for d in dirs if not d.startswith(".")] + for file in sorted(files): + file_path = os.path.join(root, file) + hasher.update(file.encode("utf-8")) + try: + with open(file_path, "rb") as f: + while chunk := f.read(8192): + hasher.update(chunk) + except Exception: + pass + + return hasher.hexdigest() + + def is_cached(self, path): + if not os.path.exists(path): + return False + return self.cache.get(path) == self.get_hash(path) + + def update(self, path): + if os.path.exists(path): + self.cache[path] = self.get_hash(path) + + def save(self): + os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) + with open(CACHE_FILE, "w", encoding="utf-8") as f: + json.dump(self.cache, f, indent=2) diff --git a/.github/scripts/validate.py b/.github/scripts/validate.py new file mode 100644 index 00000000..ebe99a70 --- /dev/null +++ b/.github/scripts/validate.py @@ -0,0 +1,58 @@ +import os +import sys +import json +import glob +from concurrent.futures import ThreadPoolExecutor +from utils.cache_manager import ValidationCache + +cache = ValidationCache() + +IGNORE_PATHS = {".git", ".github", ".cache", "node_modules", "scripts", "package-lock.json", "package.json"} + +def validate_plugin(plugin_path): + base_name = os.path.basename(plugin_path) + if base_name in IGNORE_PATHS or not os.path.exists(plugin_path): + return True, f"Skipped (system path): {base_name}" + + if cache.is_cached(plugin_path): + return True, f"Skipped (cached): {base_name}" + + errors = [] + if os.path.isdir(plugin_path): + json_files = glob.glob(os.path.join(plugin_path, "**/*.json"), recursive=True) + for jf in json_files: + try: + with open(jf, "r", encoding="utf-8") as f: + json.load(f) + except Exception as e: + errors.append(f"Invalid JSON format in {jf}: {str(e)}") + + if errors: + return False, "\n".join(errors) + + cache.update(plugin_path) + return True, f"Validated: {base_name}" + +def main(): + target_dir = os.getcwd() + entries = [os.path.join(target_dir, d) for d in os.listdir(target_dir) if d not in IGNORE_PATHS] + + print(f"🔍 Validating plugins across {len(entries)} target paths using parallel workers...") + + failed = False + with ThreadPoolExecutor(max_workers=8) as executor: + results = executor.map(validate_plugin, entries) + for success, msg in results: + print(f" -> {msg}") + if not success: + failed = True + + cache.save() + if failed: + print("❌ Plugin validation failed!") + sys.exit(1) + else: + print("✅ All plugin configurations validated successfully.") + +if __name__ == "__main__": + main() diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml new file mode 100644 index 00000000..c33bc700 --- /dev/null +++ b/.github/workflows/validate.yml @@ -0,0 +1,30 @@ +name: Validate Plugins + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + validate: + runs-on: ubuntu-latest + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Restore Validation Cache + uses: actions/cache@v4 + with: + path: .cache/plugin_validation.json + key: plugin-cache-${{ runner.os }}-${{ github.sha }} + restore-keys: | + plugin-cache-${{ runner.os }}- + + - name: Run Plugin Validator + run: python3 .github/scripts/validate.py diff --git a/.gitignore b/.gitignore index 43cc77f4..bfaa8f84 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,6 @@ build/ # Logs *.log + +# Local validation cache +.cache/ diff --git a/README.md b/README.md index d600b7c6..d635dff3 100644 --- a/README.md +++ b/README.md @@ -64,3 +64,10 @@ plugins/ ## License MIT + +## Local Validation + +Run parallel validator locally: +```bash +python3 .github/scripts/validate.py +``` diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 00000000..07be5412 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,6 @@ +{ + "name": "plugins", + "lockfileVersion": 3, + "requires": true, + "packages": {} +} From 2686089446362026a82abc90a39f5670a321feee Mon Sep 17 00:00:00 2001 From: Louis1125 Date: Sat, 22 Aug 2026 23:37:29 +0800 Subject: [PATCH 2/3] fix(ci): handle hidden manifests, resolve hash non-determinism, and update cache keys --- .github/scripts/utils/cache_manager.py | 48 +++++++------------------- .github/scripts/validate.py | 2 +- .github/workflows/validate.yml | 4 +-- 3 files changed, 15 insertions(+), 39 deletions(-) diff --git a/.github/scripts/utils/cache_manager.py b/.github/scripts/utils/cache_manager.py index 10fc1c4b..8ba088a4 100644 --- a/.github/scripts/utils/cache_manager.py +++ b/.github/scripts/utils/cache_manager.py @@ -17,39 +17,15 @@ def _load_cache(self): pass return {} - def get_hash(self, target_path): - hasher = hashlib.sha256() - - if os.path.isfile(target_path): - with open(target_path, "rb") as f: - while chunk := f.read(8192): - hasher.update(chunk) - elif os.path.isdir(target_path): - for root, dirs, files in os.walk(target_path): - # Ignore .git and .cache folders - dirs[:] = [d for d in dirs if not d.startswith(".")] - for file in sorted(files): - file_path = os.path.join(root, file) - hasher.update(file.encode("utf-8")) - try: - with open(file_path, "rb") as f: - while chunk := f.read(8192): - hasher.update(chunk) - except Exception: - pass - - return hasher.hexdigest() - - def is_cached(self, path): - if not os.path.exists(path): - return False - return self.cache.get(path) == self.get_hash(path) - - def update(self, path): - if os.path.exists(path): - self.cache[path] = self.get_hash(path) - - def save(self): - os.makedirs(os.path.dirname(CACHE_FILE), exist_ok=True) - with open(CACHE_FILE, "w", encoding="utf-8") as f: - json.dump(self.cache, f, indent=2) + def get_hash(directory_path: str) -> str: + hasher = hashlib.sha256() + for root, dirs, files in os.walk(directory_path): + dirs.sort() + dirs[:] = [d for d in dirs if d not in {".git", ".cache", "__pycache__"}] + for file in sorted(files): + full_path = os.path.join(root, file) + rel_path = os.path.relpath(full_path, directory_path) + hasher.update(rel_path.encode("utf-8")) + with open(full_path, "rb") as f: + while chunk := f.read(8192): hasher.update(chunk) + return hasher.hexdigest() \ No newline at end of file diff --git a/.github/scripts/validate.py b/.github/scripts/validate.py index ebe99a70..444c0836 100644 --- a/.github/scripts/validate.py +++ b/.github/scripts/validate.py @@ -19,7 +19,7 @@ def validate_plugin(plugin_path): errors = [] if os.path.isdir(plugin_path): - json_files = glob.glob(os.path.join(plugin_path, "**/*.json"), recursive=True) + json_files = glob.glob(os.path.join(plugin_path, "**/*.json"), recursive=True, include_hidden=True) for jf in json_files: try: with open(jf, "r", encoding="utf-8") as f: diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index c33bc700..f4b2bec8 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -22,9 +22,9 @@ jobs: uses: actions/cache@v4 with: path: .cache/plugin_validation.json - key: plugin-cache-${{ runner.os }}-${{ github.sha }} + key: plugin-cache-${{ runner.os }}-${{ hashFiles('.github/scripts/**') }}-${{ hashFiles('.github/scripts/**') }}-${{ github.sha }} restore-keys: | - plugin-cache-${{ runner.os }}- + plugin-cache-${{ runner.os }}-${{ hashFiles('.github/scripts/**') }}- - name: Run Plugin Validator run: python3 .github/scripts/validate.py From 388ea8c33d73257466df30a0f0dd71350f04b340 Mon Sep 17 00:00:00 2001 From: Louis1125 Date: Sat, 22 Aug 2026 23:46:12 +0800 Subject: [PATCH 3/3] fix(ci): restore complete ValidationCache class and methods --- .github/scripts/utils/cache_manager.py | 49 ++++++++++++++++---------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/.github/scripts/utils/cache_manager.py b/.github/scripts/utils/cache_manager.py index 8ba088a4..8bbcb19d 100644 --- a/.github/scripts/utils/cache_manager.py +++ b/.github/scripts/utils/cache_manager.py @@ -1,23 +1,8 @@ -import os import json import hashlib +import os -CACHE_FILE = os.path.join(os.getcwd(), ".cache", "plugin_validation.json") - -class ValidationCache: - def __init__(self): - self.cache = self._load_cache() - - def _load_cache(self): - if os.path.exists(CACHE_FILE): - try: - with open(CACHE_FILE, "r", encoding="utf-8") as f: - return json.load(f) - except Exception: - pass - return {} - - def get_hash(directory_path: str) -> str: +def get_hash(directory_path: str) -> str: hasher = hashlib.sha256() for root, dirs, files in os.walk(directory_path): dirs.sort() @@ -27,5 +12,31 @@ def get_hash(directory_path: str) -> str: rel_path = os.path.relpath(full_path, directory_path) hasher.update(rel_path.encode("utf-8")) with open(full_path, "rb") as f: - while chunk := f.read(8192): hasher.update(chunk) - return hasher.hexdigest() \ No newline at end of file + while chunk := f.read(8192): + hasher.update(chunk) + return hasher.hexdigest() + +class ValidationCache: + def __init__(self, cache_file: str = ".cache/plugin_validation.json"): + self.cache_file = cache_file + self.cache = self._load_cache() + + def _load_cache(self) -> dict: + if os.path.exists(self.cache_file): + try: + with open(self.cache_file, "r") as f: + return json.load(f) + except Exception: + return {} + return {} + + def is_cached(self, plugin_name: str, current_hash: str) -> bool: + return self.cache.get(plugin_name) == current_hash + + def update(self, plugin_name: str, current_hash: str) -> None: + self.cache[plugin_name] = current_hash + + def save(self) -> None: + os.makedirs(os.path.dirname(self.cache_file), exist_ok=True) + with open(self.cache_file, "w") as f: + json.dump(self.cache, f, indent=2)