Temporary: scan recent WCA assignment data

.github/workflows/wca-assignment-retry.yml

@@ -0,0 +1,128 @@
+name: Temporary WCA assignment retry
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/wca-assignment-retry.yml
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  retry:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    steps:
+      - name: Download initial scan
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh run download 30577152714 --repo coder13/Competitor-groups --name wca-assignment-scan-2026-07-30 --dir input
+
+      - name: Retry rate-limited WCIF requests
+        shell: python
+        run: |
+          import csv, json, os, random, time, urllib.error, urllib.request
+          from collections import Counter
+          from concurrent.futures import ThreadPoolExecutor, as_completed
+
+          API = "https://www.worldcubeassociation.org/api/v0"
+          HEADERS = {"User-Agent": "competitiongroups.com assignment usage analysis retry"}
+          OUT = "assignment-retry"
+          os.makedirs(OUT, exist_ok=True)
+
+          with open("input/all_competitions.csv", newline="", encoding="utf-8") as source:
+              failed = [row for row in csv.DictReader(source) if row["wcif_status"] != "ok"]
+          print(f"Retrying {len(failed)} competitions", flush=True)
+
+          def get_json(url, attempts=10):
+              last = None
+              for attempt in range(attempts):
+                  try:
+                      request = urllib.request.Request(url, headers=HEADERS)
+                      with urllib.request.urlopen(request, timeout=45) as response:
+                          return json.loads(response.read().decode(response.headers.get_content_charset() or "utf-8"))
+                  except urllib.error.HTTPError as error:
+                      last = error
+                      if error.code in (404, 422):
+                          raise
+                      retry_after = error.headers.get("Retry-After") if error.headers else None
+                      delay = float(retry_after) if retry_after and retry_after.isdigit() else min(3 * (2 ** attempt), 90)
+                      time.sleep(delay + random.random())
+                  except (urllib.error.URLError, TimeoutError) as error:
+                      last = error
+                      time.sleep(min(2 ** attempt, 30) + random.random())
+              raise last
+
+          def scan(row):
+              cid = row["competition_id"]
+              result = {
+                  "competition_id": cid,
+                  "persons_total": 0,
+                  "persons_with_assignments": 0,
+                  "assignment_count": 0,
+                  "competitor_assignment_count": 0,
+                  "staff_assignment_count": 0,
+                  "other_assignment_count": 0,
+                  "assignment_codes": "",
+                  "has_person_assignments": False,
+                  "has_competitor_assignments": False,
+                  "has_staff_assignments": False,
+                  "wcif_status": "ok",
+                  "inference": "no assignment data",
+              }
+              try:
+                  wcif = get_json(f"{API}/competitions/{cid}/wcif/public")
+                  persons = wcif.get("persons") or []
+                  result["persons_total"] = len(persons)
+                  codes = Counter()
+                  for person in persons:
+                      assignments = person.get("assignments") or []
+                      result["persons_with_assignments"] += bool(assignments)
+                      for assignment in assignments:
+                          codes[assignment.get("assignmentCode") or "unknown"] += 1
+                  result["assignment_count"] = sum(codes.values())
+                  result["competitor_assignment_count"] = codes.get("competitor", 0)
+                  result["staff_assignment_count"] = sum(v for k, v in codes.items() if k.startswith("staff-"))
+                  result["other_assignment_count"] = result["assignment_count"] - result["competitor_assignment_count"] - result["staff_assignment_count"]
+                  result["assignment_codes"] = ";".join(f"{k}:{v}" for k, v in sorted(codes.items()))
+                  result["has_person_assignments"] = result["assignment_count"] > 0
+                  result["has_competitor_assignments"] = result["competitor_assignment_count"] > 0
+                  result["has_staff_assignments"] = result["staff_assignment_count"] > 0
+                  if result["has_person_assignments"]:
+                      result["inference"] = "potential Competition Groups user: assignment data present"
+              except urllib.error.HTTPError as error:
+                  result["wcif_status"] = f"http_{error.code}"
+              except Exception as error:
+                  result["wcif_status"] = f"error:{type(error).__name__}"
+              time.sleep(0.35 + random.random() * 0.3)
+              return result
+
+          results = []
+          with ThreadPoolExecutor(max_workers=2) as executor:
+              futures = [executor.submit(scan, row) for row in failed]
+              for index, future in enumerate(as_completed(futures), 1):
+                  results.append(future.result())
+                  if index % 10 == 0 or index == len(futures):
+                      print(f"Retried {index}/{len(futures)}", flush=True)
+
+          fields = list(results[0]) if results else []
+          with open(f"{OUT}/retry_results.csv", "w", newline="", encoding="utf-8") as output:
+              writer = csv.DictWriter(output, fieldnames=fields)
+              writer.writeheader(); writer.writerows(sorted(results, key=lambda row: row["competition_id"]))
+          summary = {
+              "retried": len(results),
+              "successful": sum(row["wcif_status"] == "ok" for row in results),
+              "remaining_failures": dict(Counter(row["wcif_status"] for row in results if row["wcif_status"] != "ok")),
+              "with_assignments": sum(bool(row["has_person_assignments"]) for row in results),
+          }
+          with open(f"{OUT}/retry_summary.json", "w", encoding="utf-8") as output:
+              json.dump(summary, output, indent=2)
+          print(json.dumps(summary, indent=2), flush=True)
+
+      - name: Upload retry results
+        uses: actions/upload-artifact@v4
+        with:
+          name: wca-assignment-retry-2026-07-30
+          path: assignment-retry/
+          retention-days: 7

.github/workflows/wca-assignment-scan-v2.yml

@@ -0,0 +1,160 @@
+name: Temporary WCA assignment scan v2
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/wca-assignment-scan-v2.yml
+
+permissions:
+  contents: read
+
+jobs:
+  scan:
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+      - name: Scan public WCIF assignment data
+        shell: python
+        run: |
+          import csv, json, os, time, urllib.error, urllib.parse, urllib.request
+          from collections import Counter
+          from concurrent.futures import ThreadPoolExecutor, as_completed
+          from datetime import date
+
+          START = date.fromisoformat("2026-04-30")
+          END = date.fromisoformat("2026-07-30")
+          API = "https://www.worldcubeassociation.org/api/v0"
+          HEADERS = {"User-Agent": "competitiongroups.com assignment usage analysis"}
+          OUT = "assignment-scan-v2"
+          os.makedirs(OUT, exist_ok=True)
+
+          def get_json(url, attempts=4):
+              last = None
+              for attempt in range(attempts):
+                  try:
+                      request = urllib.request.Request(url, headers=HEADERS)
+                      with urllib.request.urlopen(request, timeout=30) as response:
+                          return json.loads(response.read().decode(response.headers.get_content_charset() or "utf-8"))
+                  except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as error:
+                      last = error
+                      if getattr(error, "code", None) in (404, 422):
+                          raise
+                      time.sleep(min(2 ** attempt, 8))
+              raise last
+
+          competitions = []
+          page_signatures = set()
+          for page in range(1, 101):
+              query = urllib.parse.urlencode({"start": START.isoformat(), "end": END.isoformat(), "page": page})
+              batch = get_json(f"{API}/competitions?{query}")
+              if not batch:
+                  break
+              signature = tuple(item["id"] for item in batch)
+              if signature in page_signatures:
+                  print(f"Duplicate page detected at page {page}; stopping pagination")
+                  break
+              page_signatures.add(signature)
+              competitions.extend(batch)
+              print(f"Competition page {page}: {len(batch)}", flush=True)
+              if len(batch) < 25:
+                  break
+          else:
+              raise RuntimeError("Competition pagination exceeded 100 pages")
+
+          competitions = {
+              item["id"]: item for item in competitions
+              if START <= date.fromisoformat(item["start_date"]) <= END
+          }
+          competitions = sorted(competitions.values(), key=lambda item: (item["start_date"], item["id"]))
+          print(f"Competitions in final date range: {len(competitions)}", flush=True)
+
+          def scan(item):
+              cid = item["id"]
+              row = {
+                  "competition_id": cid,
+                  "name": item.get("name", ""),
+                  "start_date": item.get("start_date", ""),
+                  "end_date": item.get("end_date", ""),
+                  "country_iso2": item.get("country_iso2", ""),
+                  "city": item.get("city", ""),
+                  "cancelled": bool(item.get("cancelled_at")),
+                  "persons_total": 0,
+                  "persons_with_assignments": 0,
+                  "assignment_count": 0,
+                  "competitor_assignment_count": 0,
+                  "staff_assignment_count": 0,
+                  "other_assignment_count": 0,
+                  "assignment_codes": "",
+                  "has_person_assignments": False,
+                  "has_competitor_assignments": False,
+                  "has_staff_assignments": False,
+                  "wcif_status": "ok",
+                  "inference": "no assignment data",
+                  "wca_url": item.get("url") or f"https://www.worldcubeassociation.org/competitions/{cid}",
+                  "competitiongroups_url": f"https://www.competitiongroups.com/competitions/{cid}",
+              }
+              try:
+                  wcif = get_json(f"{API}/competitions/{cid}/wcif/public")
+                  persons = wcif.get("persons") or []
+                  row["persons_total"] = len(persons)
+                  codes = Counter()
+                  for person in persons:
+                      assignments = person.get("assignments") or []
+                      row["persons_with_assignments"] += bool(assignments)
+                      for assignment in assignments:
+                          codes[assignment.get("assignmentCode") or "unknown"] += 1
+                  row["assignment_count"] = sum(codes.values())
+                  row["competitor_assignment_count"] = codes.get("competitor", 0)
+                  row["staff_assignment_count"] = sum(v for k, v in codes.items() if k.startswith("staff-"))
+                  row["other_assignment_count"] = row["assignment_count"] - row["competitor_assignment_count"] - row["staff_assignment_count"]
+                  row["assignment_codes"] = ";".join(f"{k}:{v}" for k, v in sorted(codes.items()))
+                  row["has_person_assignments"] = row["assignment_count"] > 0
+                  row["has_competitor_assignments"] = row["competitor_assignment_count"] > 0
+                  row["has_staff_assignments"] = row["staff_assignment_count"] > 0
+                  if row["has_person_assignments"]:
+                      row["inference"] = "potential Competition Groups user: assignment data present"
+              except urllib.error.HTTPError as error:
+                  row["wcif_status"] = f"http_{error.code}"
+              except Exception as error:
+                  row["wcif_status"] = f"error:{type(error).__name__}"
+              return row
+
+          rows = []
+          with ThreadPoolExecutor(max_workers=20) as executor:
+              futures = [executor.submit(scan, item) for item in competitions]
+              for index, future in enumerate(as_completed(futures), 1):
+                  rows.append(future.result())
+                  if index % 50 == 0 or index == len(futures):
+                      print(f"Scanned {index}/{len(futures)}", flush=True)
+
+          rows.sort(key=lambda item: (item["start_date"], item["competition_id"]))
+          assignment_rows = [row for row in rows if row["has_person_assignments"]]
+          fields = list(rows[0]) if rows else []
+          for filename, output_rows in (("all_competitions.csv", rows), ("competitions_with_assignments.csv", assignment_rows)):
+              with open(f"{OUT}/{filename}", "w", newline="", encoding="utf-8") as output:
+                  writer = csv.DictWriter(output, fieldnames=fields)
+                  writer.writeheader(); writer.writerows(output_rows)
+
+          summary = {
+              "date_window": {"start": START.isoformat(), "end": END.isoformat(), "basis": "competition start_date"},
+              "competitions_total": len(rows),
+              "competitions_cancelled": sum(row["cancelled"] for row in rows),
+              "competitions_with_assignments": len(assignment_rows),
+              "assignment_coverage_percent": round(100 * len(assignment_rows) / len(rows), 2) if rows else 0,
+              "competitions_with_competitor_assignments": sum(row["has_competitor_assignments"] for row in rows),
+              "competitions_with_staff_assignments": sum(row["has_staff_assignments"] for row in rows),
+              "total_person_assignments": sum(row["assignment_count"] for row in rows),
+              "wcif_failures": dict(Counter(row["wcif_status"] for row in rows if row["wcif_status"] != "ok")),
+              "by_country": dict(Counter(row["country_iso2"] for row in assignment_rows).most_common()),
+              "by_month": dict(sorted(Counter(row["start_date"][:7] for row in assignment_rows).items())),
+          }
+          with open(f"{OUT}/summary.json", "w", encoding="utf-8") as output:
+              json.dump(summary, output, indent=2, ensure_ascii=False)
+          print(json.dumps(summary, indent=2), flush=True)
+
+      - name: Upload scan results
+        uses: actions/upload-artifact@v4
+        with:
+          name: wca-assignment-scan-v2-2026-07-30
+          path: assignment-scan-v2/
+          retention-days: 7