Skip to content

Commit 736434f

Browse files
committed
fix(gitlab): reject inconsistent membership pagination totals
1 parent 86bd545 commit 736434f

3 files changed

Lines changed: 18 additions & 3 deletions

File tree

‎apps/docs/content/docs/search/gitlab.mdx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ This creates `gitlab-permissions/users.csv` and `gitlab-permissions/project-perm
102102

103103
The export includes active, unlocked human members with Reporter access or higher. It excludes Guests, Planners, bots, and memberships expiring today or earlier in UTC. It does not infer access from public/internal visibility, administrator or Auditor privileges, or custom permissions below Reporter. GitLab policies such as SSO or external authorization are not represented by these CSVs. Review the grants against your organization's access requirements before uploading them.
104104

105-
Requests run serially, paced at 60 per minute by default. Set `--requests-per-minute` to change this export-job ceiling; it does not configure live Search traffic. Each run uses one project request, one request per membership page, and one profile request per candidate reader. API errors, hidden emails, identity conflicts, and oversized exports stop the job without publishing either file.
105+
Requests run serially, paced at 60 per minute by default. Set `--requests-per-minute` to change this export-job ceiling; it does not configure live Search traffic. Each run uses one project request, one request per membership page, and one profile request per candidate reader. API errors, hidden emails, identity conflicts, inconsistent pagination totals (when GitLab provides them), and oversized exports stop the job without publishing either file. Run exports after bulk membership updates finish; paginated API reads are not a point-in-time snapshot.
106106

107107
## Access checks
108108

‎apps/docs/public/static/examples/gitlab/export-permissions.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,11 +100,19 @@ def export(client, project):
100100
users, emails, seen_members = {}, {}, set()
101101
today = datetime.now(timezone.utc).date()
102102
page = 1
103+
expected_total = None
103104
while page <= 1000:
104105
members, headers = client.get(f"projects/{project_id}/members/all",
105106
{"per_page": 100, "page": page})
106107
if not isinstance(members, list) or len(members) > 100:
107108
raise ExportError("GitLab returned an invalid membership page.")
109+
total = headers.get("X-Total")
110+
if total is not None:
111+
if not re.fullmatch(r"[0-9]{1,6}", total) or int(total) > MAX_ROWS:
112+
raise ExportError("GitLab membership total is invalid or exceeds the 100,000-row limit.")
113+
if expected_total is not None and int(total) != expected_total:
114+
raise ExportError("GitLab membership total changed during pagination. Rerun the export.")
115+
expected_total = int(total)
108116
for member in members:
109117
if not isinstance(member, dict):
110118
raise ExportError("GitLab returned an invalid membership.")
@@ -154,6 +162,8 @@ def export(client, project):
154162
page = int(next_page)
155163
else:
156164
raise ExportError("GitLab membership pagination exceeds 1,000 pages.")
165+
if expected_total is not None and len(seen_members) != expected_total:
166+
raise ExportError("Exported membership count does not match GitLab's total. Rerun the export.")
157167
if not users:
158168
raise ExportError("No eligible readers. Pause search in Sim; empty CSVs cannot revoke all grants.")
159169
return project_path, users

‎apps/docs/scripts/test-gitlab-export-e2e.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,10 @@ def do_GET(self):
5151
"issues_access_level": "enabled", "wiki_access_level": "enabled"})
5252
elif endpoint == "projects/91/members/all":
5353
page = parse_qs(parsed.query).get("page", ["1"])[0]
54+
total = "11" if scenario in ("truncated-membership", "changing-total") else "10"
55+
if scenario == "changing-total" and page == "2":
56+
total = "10"
57+
headers = {} if scenario == "missing-pagination" else {"X-Total": total}
5458
if scenario == "page-failure" and page == "2":
5559
self.reply(500, {"message": TOKEN})
5660
elif scenario == "pagination-loop":
@@ -60,13 +64,13 @@ def do_GET(self):
6064
elif scenario == "empty":
6165
self.reply(200, [member(3, access_level=10)])
6266
elif page == "1":
63-
self.reply(200, [member(2), member(1)], {"X-Next-Page": "2"})
67+
self.reply(200, [member(2), member(1)], {**headers, "X-Next-Page": "2"})
6468
else:
6569
self.reply(200, [member(3, access_level=10), member(4, state="blocked"),
6670
member(5), member(6, expires_at="2000-01-01"),
6771
member(7), member(8, access_level=15), member(9),
6872
member(10, expires_at=datetime.now(timezone.utc).date().isoformat())],
69-
{"X-Next-Page": ""})
73+
{**headers, "X-Next-Page": ""})
7074
elif endpoint.startswith("users/"):
7175
user_id = int(endpoint.split("/")[1])
7276
email = {1: "ALICE@example.com", 2: "bob@example.com", 7: "carol@example.com"}.get(
@@ -117,6 +121,7 @@ def run():
117121
for scenario, expected_error in [
118122
("success", None), ("numeric-project", None), ("missing-pagination", None),
119123
("issues-only", None), ("paced", None),
124+
("truncated-membership", "total"), ("changing-total", "changed"),
120125
("hidden-email", "email"), ("duplicate-email", "email"),
121126
("wrong-identity", "identity"), ("denied", "403"),
122127
("page-failure", "500"), ("pagination-loop", "pagination"),

0 commit comments

Comments
 (0)