From b9646eac75a8aedf923e9f4f7c086eb017659841 Mon Sep 17 00:00:00 2001 From: richarddushime Date: Sun, 5 Jul 2026 11:41:32 +0200 Subject: [PATCH] build publications --- .claude/settings.local.json | 16 +- .github/workflows/README.md | 15 ++ .github/workflows/update_publications.yml | 90 +++++++ .gitignore | 3 + content/publications/README | 55 +++- content/publications/template.yaml | 17 ++ requirements.txt | 1 + scripts/build_publications.py | 312 ++++++++++++++++++++++ 8 files changed, 506 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/update_publications.yml create mode 100644 scripts/build_publications.py diff --git a/.claude/settings.local.json b/.claude/settings.local.json index bfd876763b0..b4b6b015c51 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -3,7 +3,21 @@ "allow": [ "Bash(wc:*)", "Bash(git:*)", - "Bash(gh pr:*)" + "Bash(gh pr:*)", + "Bash(python3 -c \"import yaml; print\\(yaml.__version__\\)\")", + "Bash(pip3 show *)", + "Bash(pip3 install *)", + "Bash(python3 -c \"from ruamel.yaml import YAML; print\\('ok'\\)\")", + "Bash(mkdir -p /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest/data /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest/scripts)", + "Bash(cp /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/publications_test.yaml /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest/data/publications.yaml)", + "Bash(cp /Users/rdm/oss/forrtproject.github.io/scripts/build_publications.py /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest/scripts/)", + "Bash(python3 scripts/build_publications.py)", + "Bash(mkdir -p /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest2/data /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest2/scripts)", + "Bash(cp /Users/rdm/oss/forrtproject.github.io/scripts/build_publications.py /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest2/scripts/)", + "Bash(cat)", + "Bash(mkdir -p /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest3/data /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest3/scripts)", + "Bash(cp /Users/rdm/oss/forrtproject.github.io/scripts/build_publications.py /private/tmp/claude-501/-Users-rdm-oss-forrtproject-github-io/d40a3858-03b0-4abe-8e07-eb52ecb3b04b/scratchpad/pubtest3/scripts/)", + "Bash(python3 scripts/build_publications.py --dry-run)" ] } } diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 04c704ec6a4..01837eb766b 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -72,6 +72,21 @@ Processes: Triggers `deploy.yaml` after successful processing via `repository_dispatch`. +### [update_publications.yml](workflows/update_publications.yml) +**Hydrates DOI-stub entries in `data/publications.yaml`** + +| Trigger | Schedule | +|---------|----------| +| Push to `main` touching `data/publications.yaml` | On push | +| Weekly | Monday 03:30 UTC | +| Manual dispatch | Manual (`force` option to bypass the DOI cache) | + +Runs `scripts/build_publications.py`, which fetches title/authors/journal/year/ +abstract/citation for any entry with a top-level `doi:` key from Crossref/DataCite +(via doi.org content negotiation). Never commits to `main` directly — opens a PR +for review when metadata changes, same pattern as the glossary-update PR in +`data-processing.yml`. See `content/publications/README` for the entry format. + ## Quality Checks ### [spell-check.yaml](workflows/spell-check.yaml) diff --git a/.github/workflows/update_publications.yml b/.github/workflows/update_publications.yml new file mode 100644 index 00000000000..04876c52e4e --- /dev/null +++ b/.github/workflows/update_publications.yml @@ -0,0 +1,90 @@ +name: Update publications + +# ===================================================================== +# Hydrates DOI-stub entries in data/publications.yaml (see +# scripts/build_publications.py) by fetching title/authors/journal/year/ +# abstract/citation from the DOI's registration agency (Crossref/DataCite). +# +# Never commits to main directly: if hydration produces a diff, it's pushed +# to a new branch and opened as a PR for human review before it goes live. +# +# Triggers: +# - push to main touching data/publications.yaml (hydrate a newly added +# DOI stub right away) +# - weekly schedule (catch metadata drift: preprint -> published, +# volume/page assignment, etc.) +# - workflow_dispatch (manual run, optionally ignoring the DOI cache) +# ===================================================================== + +on: + push: + branches: [main] + paths: + - 'data/publications.yaml' + schedule: + - cron: '30 3 * * 1' # weekly, Monday 03:30 UTC + workflow_dispatch: + inputs: + force: + description: 'Ignore the DOI lookup cache and refetch everything' + required: false + type: boolean + default: false + +concurrency: + group: update-publications + cancel-in-progress: false + +jobs: + hydrate: + name: Hydrate publication DOIs + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + run: pip install ruamel.yaml + + - name: Restore DOI metadata cache + uses: actions/cache@v4 + with: + path: scripts/publications_doi_cache.json + key: publications-doi-cache-${{ github.run_id }} + restore-keys: | + publications-doi-cache- + + - name: Run hydration script + run: | + python3 scripts/build_publications.py ${{ inputs.force && '--force' || '' }} + + - name: Create PR with hydrated metadata + env: + GITHUB_TOKEN: ${{ secrets.FORRT_PAT }} + run: | + if [ -z "$(git status --porcelain -- data/publications.yaml)" ]; then + echo "ℹī¸ No publication metadata changes detected" + exit 0 + fi + + BRANCH_NAME="publications-update-$(date +%Y%m%d-%H%M%S)" + git checkout -b "$BRANCH_NAME" + + git add data/publications.yaml + git commit -m "Update hydrated publication metadata - $(date -u +'%Y-%m-%d %H:%M:%S UTC')" + + git push -u origin "$BRANCH_NAME" + + gh pr create \ + --title "📚 Publications metadata update - $(date '+%Y-%m-%d')" \ + --body "Automated DOI metadata hydration generated on $(date -u +'%Y-%m-%d %H:%M:%S UTC') by scripts/build_publications.py. Please review the hydrated title/authors/abstract/citation fields before merging." \ + --base main \ + --head "$BRANCH_NAME" + echo "✅ PR created for publications update" diff --git a/.gitignore b/.gitignore index 92078259524..61ce9f9f455 100644 --- a/.gitignore +++ b/.gitignore @@ -277,3 +277,6 @@ bibtex_to_apa/node_modules/ # DOI lookup cache for cluster parsing script scripts/doi_cache.json + +# DOI metadata cache for publications hydration script +scripts/publications_doi_cache.json diff --git a/content/publications/README b/content/publications/README index 6ef1564c72a..61c882dae82 100644 --- a/content/publications/README +++ b/content/publications/README @@ -2,9 +2,60 @@ This folder contains PDF files and other publication-related assets. The actual publication data is stored in `/data/publications.yaml` (NOT in this folder). -## Adding a New Publication +## Adding a New Publication (DOI, recommended) -Publications are managed in the **`/data/publications.yaml`** file. This file contains all publication metadata that is displayed on the website. +If the publication has a DOI, you only need to add a minimal stub — a script +fetches the rest. Add an entry with a top-level `doi:` key: + +```yaml +- doi: "10.1016/j.socec.2025.102502" + type: "journal" # journal, preprint, media, policy, affiliated, wip + focus_area: "Meta-Research & Scientific Reform" +``` + +Commit that stub to `data/publications.yaml` and push (or open a PR). The +[update_publications.yml](/.github/workflows/update_publications.yml) workflow +runs `scripts/build_publications.py`, which resolves the DOI via Crossref/DataCite +and fills in `title`, `authors`, `journal_name`, `year`, `status`, `abstract`, +`citation`, `links.doi`, and `altmetric_doi`. It opens a PR with the hydrated +result for review — nothing is pushed straight to `main`. + +**On every run, all of the auto-filled fields above are recomputed from the DOI +and will overwrite hand edits.** To pin a field permanently (a custom abstract +blurb, an extra `postprint`/`pdf`/`project` link, a manually-chosen `status`), +add it under `overrides:` instead of editing the auto field directly: + +```yaml +- doi: "10.1016/j.socec.2025.102502" + type: "journal" + focus_area: "Meta-Research & Scientific Reform" + overrides: + abstract: "Custom short blurb instead of the paper's full abstract" + status: "In Press" + links: + postprint: "https://osf.io/preprints/..." + pdf: "./pdf/local-copy.pdf" +``` + +`type` and `focus_area` are never touched by the script — always set those by +hand. If a DOI fails to resolve (bad DOI, API outage), the script leaves that +entry's existing fields untouched and logs a warning rather than blanking them. + +Citations are built as +`{authors} ({year}). {title}. {journal_name}, {volume}({issue}), {page}. https://doi.org/{doi}` +— good for typical journal articles and preprints, but book chapters and other +unusual venues often need a hand-written `overrides.citation`. Check the PR +diff before merging. + +**Note:** Existing publications that predate this pipeline are still fully +hand-written (no top-level `doi:` key) and are left alone by the script — +migrating them to the DOI-stub format is a separate, deliberate task, not +something that happens automatically. + +## Adding a New Publication (manual, no DOI) + +For publications without a DOI (op-eds, policy briefs, work-in-progress, etc.), +write the full entry by hand as before. ### Steps to Add a Publication diff --git a/content/publications/template.yaml b/content/publications/template.yaml index 8601386e13a..692b08f8f89 100644 --- a/content/publications/template.yaml +++ b/content/publications/template.yaml @@ -1,6 +1,23 @@ # Below is a template for adding a new publication to the publications.yaml file. in data/publications.yaml # Remove or add the fields as needed following the layout of the existing entries. +# --- Preferred: DOI stub (auto-hydrated by scripts/build_publications.py) --- +# Only `doi`, `type`, and `focus_area` are hand-written; the rest is fetched +# from Crossref/DataCite and refreshed on every run. Pin a field permanently by +# adding it under `overrides:` instead. See content/publications/README. + +- doi: "10.xxxx/xxxxx" + type: "journal" # journal, preprint, media, policy, affiliated, wip + focus_area: "Meta-Research & Scientific Reform" + # overrides: + # abstract: "Custom short blurb instead of the paper's full abstract" + # status: "In Press" + # links: + # postprint: "https://osf.io/preprints/..." + # pdf: "./pdf/local-copy.pdf" + +# --- Fallback: full manual entry (no DOI, e.g. op-eds, policy briefs, WIP) --- + - title: "" type: "" year: "" diff --git a/requirements.txt b/requirements.txt index 2a95cae1bae..cc086b057f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ gspread python-dotenv matplotlib pypinyin +ruamel.yaml diff --git a/scripts/build_publications.py b/scripts/build_publications.py new file mode 100644 index 00000000000..2091ebe54dd --- /dev/null +++ b/scripts/build_publications.py @@ -0,0 +1,312 @@ +#!/usr/bin/env python3 +""" +Hydrate DOI-stub entries in data/publications.yaml with metadata fetched from +the DOI's registration agency (Crossref or DataCite, via doi.org content +negotiation). + +An entry opts into hydration by having a top-level `doi:` key, e.g.: + + - doi: "10.1016/j.socec.2025.102502" + type: "journal" + focus_area: "Meta-Research & Scientific Reform" + +On every run, the script (re)computes title, authors, journal_name, year, +status, abstract, citation, links.doi, and altmetric_doi for every such entry. +Any field can be permanently pinned by adding it under `overrides:` (nested +dicts, e.g. overrides.links.pdf, are deep-merged). `type` and `focus_area` are +never touched by this script. + +Entries without a top-level `doi:` key (today's media/policy/wip entries) are +left completely untouched. + +If a DOI fails to resolve, the entry's existing fields are left as-is (so a +transient API outage never blanks out already-good data) and a warning is +printed. + +Usage: + python3 scripts/build_publications.py [--dry-run] [--force] +""" + +import argparse +import html +import json +import re +import sys +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import quote +from urllib.request import Request, urlopen + +from ruamel.yaml import YAML +from ruamel.yaml.comments import CommentedMap + +DATA_FILE = Path("data/publications.yaml") +CACHE_FILE = Path("scripts/publications_doi_cache.json") +USER_AGENT = "FORRT-PublicationsBuilder/1.0 (mailto:info@forrt.org)" + +AUTO_FIELDS = ["title", "authors", "journal_name", "year", "status", "abstract", "citation"] + +STATUS_MAP = { + "journal-article": "Published", + "posted-content": "Preprint", + "preprint": "Preprint", + "proceedings-article": "Published", + "book-chapter": "Published", + "report": "Report", + "monograph": "Published", +} + +TAG_RE = re.compile(r"<[^>]+>") +ABSTRACT_PREFIX_RE = re.compile(r"^(abstract\.?\s*[:.]?\s*)", re.IGNORECASE) +WHITESPACE_RE = re.compile(r"\s+") + + +def strip_tags(text): + if not text: + return "" + text = TAG_RE.sub(" ", text) + text = html.unescape(text) + return WHITESPACE_RE.sub(" ", text).strip() + + +def clean_abstract(text): + text = strip_tags(text) + if not text: + return "" + return ABSTRACT_PREFIX_RE.sub("", text).strip() + + +def format_initials(given): + """'Flavio' -> 'F.', 'William R.' -> 'W. R.', 'Jean-Michel' -> 'J.-M.'""" + given = (given or "").strip() + if not given: + return "" + initials = [] + for part in re.split(r"\s+", given): + sub_initials = [ + f"{sub[0].upper()}." for sub in part.split("-") if sub.strip(".") + ] + if sub_initials: + initials.append("-".join(sub_initials)) + return " ".join(initials) + + +def format_authors(csl_authors): + names = [] + for author in csl_authors or []: + family = (author.get("family") or "").strip() + if not family: + literal = (author.get("literal") or "").strip() + if literal: + names.append(literal) + continue + initials = format_initials(author.get("given")) + names.append(f"{family}, {initials}" if initials else family) + + if not names: + return "" + if len(names) == 1: + return names[0] + if len(names) > 9: + return ", ".join(names[:8]) + ", ... & " + names[-1] + return ", ".join(names[:-1]) + " & " + names[-1] + + +def derive_status(csl_type): + if not csl_type: + return "Published" + return STATUS_MAP.get(csl_type, csl_type.replace("-", " ").title()) + + +def derive_year(csl): + issued = csl.get("issued") or {} + parts = issued.get("date-parts") or [] + if parts and parts[0]: + return str(parts[0][0]) + return "" + + +def normalize_doi(doi): + doi = (doi or "").strip() + doi = re.sub(r"^https?://(dx\.)?doi\.org/", "", doi, flags=re.IGNORECASE) + return doi + + +def build_citation(authors, year, title, journal_name, volume, issue, page, doi): + citation = f"{authors} ({year}). {title}." + if journal_name: + tail = f" {journal_name}" + if volume: + tail += f", {volume}" + if issue: + tail += f"({issue})" + if page: + tail += f", {page}" + citation += tail + "." + citation += f" https://doi.org/{doi}" + return citation + + +def fetch_json(url, accept=None, timeout=20): + req = Request(url) + if accept: + req.add_header("Accept", accept) + req.add_header("User-Agent", USER_AGENT) + with urlopen(req, timeout=timeout) as resp: + return json.load(resp) + + +def fetch_csl_json(doi): + url = f"https://doi.org/{quote(doi, safe='/:@!$&()*+,;=-._~')}" + try: + return fetch_json(url, accept="application/vnd.citationstyles.csl+json") + except HTTPError as e: + print(f" WARNING: doi.org returned HTTP {e.code} for {doi}", file=sys.stderr) + except (URLError, TimeoutError, json.JSONDecodeError) as e: + print(f" WARNING: failed to resolve {doi}: {e}", file=sys.stderr) + return None + + +def fetch_datacite_abstract(doi): + url = f"https://api.datacite.org/dois/{quote(doi, safe='/:@!$&()*+,;=-._~')}" + try: + data = fetch_json(url) + except (HTTPError, URLError, TimeoutError, json.JSONDecodeError): + return "" + descriptions = (data.get("data") or {}).get("attributes", {}).get("descriptions") or [] + for d in descriptions: + if d.get("descriptionType") == "Abstract": + abstract = clean_abstract(d.get("description", "")) + if abstract: + return abstract + return "" + + +def build_metadata(doi): + csl = fetch_csl_json(doi) + if csl is None: + return None + + title = strip_tags(csl.get("title", "")) + container_title = csl.get("container-title") or "" + if isinstance(container_title, list): + container_title = container_title[0] if container_title else "" + # Repository-hosted preprints (DataCite) often have no container-title; + # fall back to the publisher (e.g. "Center for Open Science"). + journal_name = strip_tags(container_title) or strip_tags(csl.get("publisher", "")) + authors = format_authors(csl.get("author")) + year = derive_year(csl) + status = derive_status(csl.get("type")) + volume = csl.get("volume", "") + issue = csl.get("issue", "") + page = csl.get("page") or csl.get("article-number") or "" + + abstract = clean_abstract(csl.get("abstract", "")) + if not abstract: + abstract = fetch_datacite_abstract(doi) + + citation = build_citation(authors, year, title, journal_name, volume, issue, page, doi) + + return { + "doi": doi, + "title": title, + "authors": authors, + "journal_name": journal_name, + "year": year, + "status": status, + "abstract": abstract, + "citation": citation, + } + + +def load_cache(): + if CACHE_FILE.exists(): + with CACHE_FILE.open(encoding="utf-8") as f: + return json.load(f) + return {} + + +def save_cache(cache): + CACHE_FILE.parent.mkdir(parents=True, exist_ok=True) + with CACHE_FILE.open("w", encoding="utf-8") as f: + json.dump(cache, f, indent=2, ensure_ascii=False) + + +def resolve_doi(doi, cache, force=False): + if not force and doi in cache: + return cache[doi] + metadata = build_metadata(doi) + if metadata is not None: + cache[doi] = metadata + return metadata + + +def deep_merge_links(entry, doi, overrides): + links = entry.get("links") + if links is None: + links = CommentedMap() + entry["links"] = links + links["doi"] = f"https://doi.org/{doi}" + for key, value in (overrides.get("links") or {}).items(): + links[key] = value + + +def hydrate_entry(entry, metadata): + overrides = entry.get("overrides") or {} + for field in AUTO_FIELDS: + if field in overrides: + entry[field] = overrides[field] + continue + new_value = metadata.get(field) + if new_value: + entry[field] = new_value + deep_merge_links(entry, metadata["doi"], overrides) + entry["altmetric_doi"] = metadata["doi"] + + +def make_yaml(): + yaml = YAML() + yaml.preserve_quotes = True + yaml.width = 100000 + return yaml + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--dry-run", action="store_true", help="Print result, don't write the file") + parser.add_argument("--force", action="store_true", help="Ignore cached DOI lookups") + args = parser.parse_args() + + yaml = make_yaml() + with DATA_FILE.open(encoding="utf-8") as f: + data = yaml.load(f) + + cache = load_cache() + resolved, failed = 0, 0 + + for entry in data: + if not isinstance(entry, dict) or "doi" not in entry: + continue + doi = normalize_doi(entry["doi"]) + print(f"Resolving {doi} ...") + metadata = resolve_doi(doi, cache, force=args.force) + if metadata is None: + failed += 1 + print(f" Could not resolve {doi}; leaving existing fields untouched") + continue + hydrate_entry(entry, metadata) + resolved += 1 + + save_cache(cache) + print(f"Done: {resolved} resolved, {failed} failed") + + if args.dry_run: + yaml.dump(data, sys.stdout) + return + + with DATA_FILE.open("w", encoding="utf-8") as f: + yaml.dump(data, f) + + +if __name__ == "__main__": + main()