From 149f98aed709a9d368f933cf03fe94c781fd0f36 Mon Sep 17 00:00:00 2001 From: Matthieu Baerts Date: Thu, 10 Sep 2026 10:15:39 +0200 Subject: [PATCH] Revert "pw: fetch the patches from lore instead of patchwork" This reverts commit fe67168e03b2aa1748f2d5e7072a42ac0335f6a6. This was a temporary fix because after a recent PW upgrade, Patchwork's own /mbox/ endpoints stopped working. This has been fixed now. We could keep this workaround, but it seems better to drop it not to have to rely on two different services running asynchronously: higher risk to have more failures. Signed-off-by: Matthieu Baerts --- CONFIG.rst | 22 --------------- mailbot.py | 4 ++- pw/patchwork.py | 73 ++++--------------------------------------------- pw/pw_series.py | 41 +++++++++++++++++++++++---- pw_brancher.py | 3 +- 5 files changed, 46 insertions(+), 97 deletions(-) diff --git a/CONFIG.rst b/CONFIG.rst index d3b702ec..c5985d2b 100644 --- a/CONFIG.rst +++ b/CONFIG.rst @@ -3,28 +3,6 @@ Config syntax This document describes the fields of the config file and their meaning. -patchwork -========= - -Section configuring the patchwork connection. - -archive -------- - -Patchwork's own ``/mbox/`` endpoints stopped working after one of its upgrades, -so patchwork is only used for metadata and the messages themselves are fetched -from the mailing list archive. - -``archive`` is the base URL of that archive, the message id gets appended to it -(default: ``https://lore.kernel.org/all``). - -user-agent ----------- - -The ``User-Agent`` header for all HTTP requests. lore.kernel.org rejects -requests from unknown agents with a 403, so this must be set to one of the -strings the archive recognizes. - poller ====== diff --git a/mailbot.py b/mailbot.py index ea6d400c..7192b9c8 100755 --- a/mailbot.py +++ b/mailbot.py @@ -382,7 +382,9 @@ def _resolve_thread(self, pw): self._series_id = pw_obj[0]['series'][0]['id'] - data = pw.get_mbox_by_msgid(mid) + r = requests.get(f'https://lore.kernel.org/all/{mid}/raw', + headers=http_headers) + data = r.content.decode('utf-8') msg = email.message_from_string(data, policy=default) self._series_author = msg.get('From') diff --git a/pw/patchwork.py b/pw/patchwork.py index 87beb51b..9705f4ff 100644 --- a/pw/patchwork.py +++ b/pw/patchwork.py @@ -29,60 +29,17 @@ class PatchworkPostException(Exception): pass -class PatchworkFetchException(Exception): - pass - - -def series_patches_ordered(series): - """Return the patches of a series in the order they should be applied - - Patchwork lists them in arrival order, use the n/total counter it parsed - out of the subject to put them back into the order the author intended. - """ - patches = series['patches'] - total = series['total'] - if total != len(patches): - core.log("Patch order - count does not add up?!", "") - return patches - - ordered = list(patches) - for i in range(total): - found = False - name = patches[i]['name'] - for j in range(total): - # scanning PW-parsed name - tags are separated by commas - if name.find(f" {j + 1}/{total}") >= 0 or \ - name.find(f",{j + 1}/{total}") >= 0 or \ - name.find(f"[{j + 1}/{total}") >= 0 or \ - name.find(f"0{j + 1}/{total}") >= 0: - if ordered[j] is not patches[i]: - core.log(f"Patch order - reordering {i} => {j + 1}") - ordered[j] = patches[i] - found = True - break - if not found: - core.log("Patch order - not all patches were found!", "") - return patches - return ordered - - class Patchwork(object): - # Patchwork mbox object types vs the names of the REST collections - _mbox_apis = {'cover': 'covers', 'patch': 'patches'} - def __init__(self, config): self._session = requests.Session() allowed_methods = Retry.DEFAULT_ALLOWED_METHODS | {'POST', 'PATCH'} - retry = Retry(connect=10, status=10, - status_forcelist={404, 429, 502, 503, 504}, + retry = Retry(connect=10, status=10, status_forcelist={502, 504}, allowed_methods=allowed_methods, backoff_factor=1) adapter = HTTPAdapter(max_retries=retry) self._session.mount('http://', adapter) self._session.mount('https://', adapter) self.server = config.get('patchwork', 'server') - self.archive = config.get('patchwork', 'archive', - fallback='https://lore.kernel.org/all').rstrip('/') ssl = config.getboolean('patchwork', 'use_ssl', fallback=True) self._proto = "https://" if ssl else "http://" self._token = config.get('patchwork', 'token', fallback='') @@ -113,7 +70,7 @@ def _request(self, url): try: core.log("Response data", ret.json()) except json.decoder.JSONDecodeError: - core.log("Response data", ret.content.decode('utf-8', 'replace')) + core.log("Response data", ret.content.decode()) finally: end = datetime.datetime.now() core.log("Response time GET (sec)", (end - start).total_seconds()) @@ -179,30 +136,12 @@ def get_by_msgid(self, object_type, msgid): msgid = urllib.parse.quote(msgid) return self._get(f'{object_type}/?msgid={msgid}&project={self._project}', api='').json() - # Patchwork's own /mbox/ endpoints have been serving empty responses ever - # since one of its upgrades, so the messages come from the list archive. - # Patchwork is only asked for the message ids. Note that the archive - # requires a well-known user-agent, see the 'user-agent' config option. - def get_mbox_by_msgid(self, msgid): - url = f'{self.archive}/{urllib.parse.quote(msgid.strip("<>"))}/raw' - ret = self._request(url) - if ret.status_code != 200: - raise PatchworkFetchException(url, ret) - # Archives serve the message as it was posted, which is not necessarily - # valid UTF-8. Losing a character beats blowing up the entire series. - return ret.content.decode('utf-8', 'replace') - - # Like patchwork's series mbox this contains the patches only, the cover - # letter is not part of it. - def series_to_mbox(self, series): - return ''.join([self.get_mbox_by_msgid(p['msgid']) - for p in series_patches_ordered(series)]) + def get_mbox_direct(self, url): + return self._request(url).content.decode() def get_mbox(self, object_type, identifier): - if object_type == 'series': - return self.series_to_mbox(self.get('series', identifier)) - obj = self.get(self._mbox_apis[object_type], identifier) - return self.get_mbox_by_msgid(obj['msgid']) + url = f'{self._proto}{self.server}/{object_type}/{identifier}/mbox/' + return self._request(url).content.decode() def _get(self, req, api='1.1'): if api: diff --git a/pw/pw_series.py b/pw/pw_series.py index 70840378..fdbee780 100644 --- a/pw/pw_series.py +++ b/pw/pw_series.py @@ -7,7 +7,6 @@ from core import Series from core import Patch from core import log, log_open_sec, log_end_sec -from .patchwork import series_patches_ordered # TODO: document @@ -22,7 +21,7 @@ def __init__(self, pw, pw_series): self.pull_url = None if pw_series['cover_letter']: - pw_cover_letter = pw.get_mbox_by_msgid(pw_series['cover_letter']['msgid']) + pw_cover_letter = pw.get_mbox('cover', pw_series['cover_letter']['id']) self.set_cover_letter(pw_cover_letter) elif self.pw_series['patches']: self.subject = self.pw_series['patches'][0]['name'] @@ -36,14 +35,44 @@ def __init__(self, pw, pw_series): # Fast path incomplete series if not pw_series['received_all']: for p in self.pw_series['patches']: - raw_patch = pw.get_mbox_by_msgid(p['msgid']) + raw_patch = pw.get_mbox('patch', p['id']) self.patches.append(Patch(raw_patch, p['id'])) return # Do more magic around series which are complete - for p in series_patches_ordered(self.pw_series): - raw_patch = pw.get_mbox_by_msgid(p['msgid']) - self.add_patch(Patch(raw_patch, p['id'])) + # Patchwork 2.2.2 orders them by arrival time + pids = [] + for p in self.pw_series['patches']: + pids.append(p['id']) + total = self.pw_series['total'] + if total == len(self.pw_series['patches']): + for i in range(total): + found = False + name = self.pw_series['patches'][i]['name'] + pid = self.pw_series['patches'][i]['id'] + for j in range(total): + # scanning PW-parsed name - tags are separated by commas + if name.find(f" {j + 1}/{total}") >= 0 or \ + name.find(f",{j + 1}/{total}") >= 0 or \ + name.find(f"[{j + 1}/{total}") >= 0 or \ + name.find(f"0{j + 1}/{total}") >= 0: + if pids[j] != pid: + log(f"Patch order - reordering {i} => {j + 1}") + pids[j] = pid + found = True + break + if not found: + log("Patch order - not all patches were found!", "") + pids = [] + for p in self.pw_series['patches']: + pids.append(p['id']) + break + else: + log("Patch order - count does not add up?!", "") + + for pid in pids: + raw_patch = pw.get_mbox('patch', pid) + self.add_patch(Patch(raw_patch, pid)) if not pw_series['cover_letter']: if len(self.patches) == 1: diff --git a/pw_brancher.py b/pw_brancher.py index b2013e62..7e5c79f3 100755 --- a/pw_brancher.py +++ b/pw_brancher.py @@ -178,7 +178,8 @@ def apply_pending_patches(pw, config, tree, branch_name) -> Tuple[List, List]: else: log_open_sec("Applying: " + entry["series"][0]["name"]) seen_series.add(series_id) - data = pw.get_mbox('series', series_id) + mbox_url = entry["series"][0]["mbox"] + data = pw.get_mbox_direct(mbox_url) p = Patch(data) try: tree.apply(p)