|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Export one GitLab project's readers to Sim's permission CSVs (Python 3.10+).""" |
| 3 | + |
| 4 | +import argparse |
| 5 | +import csv |
| 6 | +from datetime import date, datetime, timezone |
| 7 | +import io |
| 8 | +import json |
| 9 | +import os |
| 10 | +from pathlib import Path |
| 11 | +import re |
| 12 | +import ssl |
| 13 | +import sys |
| 14 | +import tempfile |
| 15 | +import time |
| 16 | +from urllib.error import HTTPError, URLError |
| 17 | +from urllib.parse import quote, urlencode, urlsplit |
| 18 | +from urllib.request import HTTPRedirectHandler, HTTPSHandler, Request, build_opener |
| 19 | + |
| 20 | + |
| 21 | +MAX_BYTES = 4 * 1024 * 1024 |
| 22 | +MAX_ROWS = 100_000 |
| 23 | +EMAIL = re.compile( |
| 24 | + r"[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@" |
| 25 | + r"[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?" |
| 26 | + r"(?:\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*" |
| 27 | +) |
| 28 | + |
| 29 | + |
| 30 | +class ExportError(Exception): |
| 31 | + """An incomplete or unsafe export must not be uploaded.""" |
| 32 | + |
| 33 | + |
| 34 | +class NoRedirects(HTTPRedirectHandler): |
| 35 | + def redirect_request(self, req, fp, code, msg, headers, newurl): |
| 36 | + # A redirect must never forward the private token to another host. |
| 37 | + return None |
| 38 | + |
| 39 | + |
| 40 | +class GitLab: |
| 41 | + def __init__(self, base_url, token, requests_per_minute): |
| 42 | + parsed = urlsplit(base_url) |
| 43 | + if (parsed.scheme != "https" or not parsed.hostname or parsed.username |
| 44 | + or parsed.password or parsed.query or parsed.fragment |
| 45 | + or any(ord(char) < 33 for char in base_url)): |
| 46 | + raise ExportError("Use an HTTPS GitLab instance URL without credentials, query, or fragment.") |
| 47 | + self.base = base_url.rstrip("/") + "/api/v4/" |
| 48 | + self.token = token |
| 49 | + self.interval = 60 / requests_per_minute |
| 50 | + self.last_request = 0.0 |
| 51 | + self.opener = build_opener(NoRedirects(), HTTPSHandler(context=ssl.create_default_context())) |
| 52 | + |
| 53 | + def get(self, path, query=None): |
| 54 | + delay = self.interval - (time.monotonic() - self.last_request) |
| 55 | + if delay > 0: |
| 56 | + time.sleep(delay) |
| 57 | + url = self.base + path + ("?" + urlencode(query) if query else "") |
| 58 | + request = Request(url, headers={"PRIVATE-TOKEN": self.token, "Accept": "application/json"}) |
| 59 | + self.last_request = time.monotonic() |
| 60 | + try: |
| 61 | + with self.opener.open(request, timeout=30) as response: |
| 62 | + raw = response.read(8 * 1024 * 1024 + 1) |
| 63 | + headers = response.headers |
| 64 | + except HTTPError as error: |
| 65 | + raise ExportError(f"GitLab GET {path} failed (HTTP {error.code}); no export published.") from None |
| 66 | + except (URLError, TimeoutError, OSError): |
| 67 | + raise ExportError("GitLab connection failed. Check the host, HTTPS trust, and network access.") from None |
| 68 | + if len(raw) > 8 * 1024 * 1024: |
| 69 | + raise ExportError("GitLab response exceeds the 8 MiB limit.") |
| 70 | + try: |
| 71 | + return json.loads(raw), headers |
| 72 | + except (ValueError, UnicodeError): |
| 73 | + raise ExportError("GitLab returned invalid JSON.") from None |
| 74 | + |
| 75 | + |
| 76 | +def identifier(value): |
| 77 | + if type(value) is not int or not 0 < value <= 9_007_199_254_740_991: |
| 78 | + raise ExportError("GitLab returned an invalid numeric identity.") |
| 79 | + return value |
| 80 | + |
| 81 | + |
| 82 | +def active(record): |
| 83 | + if not isinstance(record, dict) or not isinstance(record.get("state"), str): |
| 84 | + raise ExportError("GitLab returned a user without an account state.") |
| 85 | + return record["state"] == "active" and not record.get("locked", False) |
| 86 | + |
| 87 | + |
| 88 | +def export(client, project): |
| 89 | + record, _ = client.get("projects/" + quote(project, safe="")) |
| 90 | + if not isinstance(record, dict): |
| 91 | + raise ExportError("GitLab returned an invalid project.") |
| 92 | + project_id = identifier(record.get("id")) |
| 93 | + project_path = record.get("path_with_namespace") |
| 94 | + if (not isinstance(project_path, str) or len(project_path) > 1024 |
| 95 | + or "/" not in project_path |
| 96 | + or any(not part or part in (".", "..") |
| 97 | + or not all(char.isalnum() or char in "_.-" for char in part) |
| 98 | + for part in project_path.split("/"))): |
| 99 | + raise ExportError("GitLab returned an invalid project path.") |
| 100 | + if record.get("repository_access_level") == "disabled": |
| 101 | + raise ExportError("The project's repository is disabled.") |
| 102 | + users, emails, seen_members = {}, {}, set() |
| 103 | + today = datetime.now(timezone.utc).date() |
| 104 | + page = 1 |
| 105 | + while page <= 1000: |
| 106 | + members, headers = client.get(f"projects/{project_id}/members/all", |
| 107 | + {"per_page": 100, "page": page}) |
| 108 | + if not isinstance(members, list) or len(members) > 100: |
| 109 | + raise ExportError("GitLab returned an invalid membership page.") |
| 110 | + for member in members: |
| 111 | + if not isinstance(member, dict): |
| 112 | + raise ExportError("GitLab returned an invalid membership.") |
| 113 | + user_id = identifier(member.get("id")) |
| 114 | + if user_id in seen_members: |
| 115 | + raise ExportError("GitLab membership pagination repeated a user. Rerun the export.") |
| 116 | + seen_members.add(user_id) |
| 117 | + if len(seen_members) > MAX_ROWS: |
| 118 | + raise ExportError("Project membership exceeds the 100,000-row limit.") |
| 119 | + role = member.get("access_level") |
| 120 | + if type(role) is not int: |
| 121 | + raise ExportError("GitLab returned a membership without an access level.") |
| 122 | + if not active(member) or role < 20: |
| 123 | + continue |
| 124 | + expiry = member.get("expires_at") |
| 125 | + if expiry is not None: |
| 126 | + try: |
| 127 | + expired = date.fromisoformat(expiry) <= today |
| 128 | + except (TypeError, ValueError): |
| 129 | + raise ExportError("GitLab returned an invalid membership expiry.") from None |
| 130 | + if expired: |
| 131 | + continue |
| 132 | + profile, _ = client.get(f"users/{user_id}") |
| 133 | + if not isinstance(profile, dict) or identifier(profile.get("id")) != user_id: |
| 134 | + raise ExportError("GitLab returned a mismatched user identity.") |
| 135 | + if not active(profile) or profile.get("bot", False): |
| 136 | + continue |
| 137 | + email = profile.get("email") |
| 138 | + if not isinstance(email, str): |
| 139 | + raise ExportError( |
| 140 | + f"User {user_id}: primary email is unavailable. The CI token needs directory email access." |
| 141 | + ) |
| 142 | + email = email.strip().lower() |
| 143 | + if len(email) > 254 or not EMAIL.fullmatch(email): |
| 144 | + raise ExportError(f"User {user_id}: primary email is invalid.") |
| 145 | + if email in emails and emails[email] != user_id: |
| 146 | + raise ExportError("Multiple GitLab users share an email. Resolve the identity conflict first.") |
| 147 | + emails[email] = user_id |
| 148 | + users[user_id] = email |
| 149 | + next_page = headers.get("X-Next-Page") |
| 150 | + if next_page is None: |
| 151 | + next_page = str(page + 1) if len(members) == 100 else "" |
| 152 | + if not next_page: |
| 153 | + break |
| 154 | + if not re.fullmatch(r"[0-9]+", next_page) or int(next_page) != page + 1 or not members: |
| 155 | + raise ExportError("GitLab returned invalid membership pagination.") |
| 156 | + page = int(next_page) |
| 157 | + else: |
| 158 | + raise ExportError("GitLab membership pagination exceeds 1,000 pages.") |
| 159 | + if not users: |
| 160 | + raise ExportError("No eligible readers. Pause search in Sim; empty CSVs cannot revoke all grants.") |
| 161 | + return project_path, users |
| 162 | + |
| 163 | + |
| 164 | +def write_export(output, project_path, users): |
| 165 | + files = {} |
| 166 | + ordered = sorted(users) |
| 167 | + for name, header, rows in [ |
| 168 | + ("users.csv", ["user_id", "email"], [(user_id, users[user_id]) for user_id in ordered]), |
| 169 | + ("project-permissions.csv", ["project_path", "user_id"], |
| 170 | + [(project_path, user_id) for user_id in ordered]), |
| 171 | + ]: |
| 172 | + buffer = io.StringIO(newline="") |
| 173 | + writer = csv.writer(buffer, lineterminator="\n") |
| 174 | + writer.writerow(header) |
| 175 | + writer.writerows(rows) |
| 176 | + files[name] = buffer.getvalue().encode("utf-8") |
| 177 | + if len(files[name]) > MAX_BYTES: |
| 178 | + raise ExportError(f"{name} exceeds Sim's 4 MiB CSV limit.") |
| 179 | + output.parent.mkdir(parents=True, exist_ok=True) |
| 180 | + # Publish the pair together, so an interrupted export cannot mix new and old CSVs. |
| 181 | + with tempfile.TemporaryDirectory(dir=output.parent, prefix=".gitlab-export-") as temporary: |
| 182 | + for name, content in files.items(): |
| 183 | + path = Path(temporary) / name |
| 184 | + path.write_bytes(content) |
| 185 | + path.chmod(0o600) |
| 186 | + os.rename(temporary, output) |
| 187 | + |
| 188 | + |
| 189 | +def main(): |
| 190 | + parser = argparse.ArgumentParser(description=__doc__) |
| 191 | + parser.add_argument("--gitlab-url", required=True, help="HTTPS instance URL, without /api/v4") |
| 192 | + parser.add_argument("--project", required=True, help="Project ID or full namespace/project path") |
| 193 | + parser.add_argument("--output-dir", default="gitlab-permissions", help="New directory for both CSVs") |
| 194 | + parser.add_argument("--requests-per-minute", type=int, default=60, help="Request-start ceiling (default: 60)") |
| 195 | + args = parser.parse_args() |
| 196 | + token = os.environ.get("GITLAB_EXPORT_TOKEN", "") |
| 197 | + if not token or any(char.isspace() for char in token): |
| 198 | + parser.error("Set GITLAB_EXPORT_TOKEN to the CI export token.") |
| 199 | + if args.requests_per_minute <= 0: |
| 200 | + parser.error("--requests-per-minute must be positive.") |
| 201 | + output = Path(args.output_dir).absolute() |
| 202 | + if output.exists() or output.is_symlink(): |
| 203 | + raise ExportError("Output directory already exists. Choose a new directory for a fresh export.") |
| 204 | + client = GitLab(args.gitlab_url, token, args.requests_per_minute) |
| 205 | + project_path, users = export(client, args.project) |
| 206 | + write_export(output, project_path, users) |
| 207 | + print(f"Exported {len(users)} readers to {output}/users.csv and {output}/project-permissions.csv") |
| 208 | + |
| 209 | + |
| 210 | +if __name__ == "__main__": |
| 211 | + try: |
| 212 | + main() |
| 213 | + except (ExportError, OSError) as error: |
| 214 | + message = str(error) if isinstance(error, ExportError) else "Could not publish the CSV files." |
| 215 | + print(f"Export failed: {message}", file=sys.stderr) |
| 216 | + sys.exit(1) |
0 commit comments