From d19a32926ccb5bd4ec0d4b96ee14c1d21cae7853 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Schnabel?= Date: Wed, 23 Sep 2026 19:49:20 +0200 Subject: [PATCH] Add gamslib regression test comparing cuOpt objectives against CPLEX baseline --- tests/baseline.txt | 73 +++++++++++++++ tests/test-gamslib-cuopt.py | 176 ++++++++++++++++++++++++++++++++++++ 2 files changed, 249 insertions(+) create mode 100644 tests/baseline.txt create mode 100644 tests/test-gamslib-cuopt.py diff --git a/tests/baseline.txt b/tests/baseline.txt new file mode 100644 index 0000000..c3509d2 --- /dev/null +++ b/tests/baseline.txt @@ -0,0 +1,73 @@ +ajax LP 441003.595310465 +alum MIP 49563.9048247556 +ampl LP 79.3412924444444 +bchtsp MIP 78 +bid MIP 15210109.512 +bidpwl MIP 15210109.512 +china LP 40561.5739263233 +clearlak LP 93.75 +coex MIP 9 +coexx MIP 4 +copper MIP 71961.2247471606 +cross MIP 3 +cube MIP 4 +cvrp MIP 502.860326158669 +demo1 LP 1898.52 +dicex MIP 21 +diet LP 0.108662278206757 +dinam LP 251.365970398347 +egypt LP 4134175.70190708 +emfl QCP 46.8675443241308 +fawley LP 2899.25279042307 +fdesign QCP 1.04648764694697 +fertd MIP 173.406208873765 +ferts LP 58793.8227006306 +food MIP 100278.703703704 +indus LP 901.161538066269 +indus89 LP 114873.655551596 +iswnm LP 114.03973 +jobt LP 21343.0555555556 +knapsack MIP 295 +korpet MIP 123054531.948789 +latin MIP 32 +magic MIP 988540 +marilyn MIP 36 +markov LP 2401.57725414587 +mexls LP 27569.5988971496 +mexsd MIP 12850.8607386776 +mexss LP 538.811203982966 +mine LP 17500 +orani LP -1.60320720847808 +pak LP 1075.5471391775 +paklive LP 19468.0876969476 +pdi LP 294070 +port LP 0.298363636363636 +poutil MIP 266793 +prodmix LP 18666.6666666667 +prodsch MIP 3254.02381132426 +prodsp LP 17869.0449355452 +qabel QCP 46965.0361591634 +qcp1 QCP 0.00341240700409326 +qdemo7 QCP 1589042.38550245 +qp5 LP 0.431455896620604 +qp7 QCP 0.000809315474402856 +qsambal QCP 3.96822036176297 +rcpsp MIP 43 +rdata MIP 3 +reaction MIP 1 +robert LP 11025 +rotdk MIP 1956.12959427829 +sarf LP 79122.6599139197 +shale LP 9354.86436047207 +springchain QCP -185.44606263468 +sroute LP 6468 +srpchase LP 0.400085953348623 +stockcc MIP 119948.688333333 +tablelayout MIP 130 +tabora LP 8471.95730320295 +thai MIP 211.111 +timesteps MIP 1007 +trnsport LP 153.675 +turkpow LP 12657.770751907 +whouse LP -600 +yemcem MIP 1620.27779181762 diff --git a/tests/test-gamslib-cuopt.py b/tests/test-gamslib-cuopt.py new file mode 100644 index 0000000..924050a --- /dev/null +++ b/tests/test-gamslib-cuopt.py @@ -0,0 +1,176 @@ +"""Regression test: solve gamslib models with cuOpt and compare against CPLEX. + +Each model listed in baseline.txt (name, model type, CPLEX objective) is +extracted with `gamslib`, solved with `solver=cuopt`, and the objective value +reported in the GAMS trace file is compared against the CPLEX reference. + +Run with: + python3 tests/test-gamslib-cuopt.py [-g GAMS_DIR] [-j JOBS] [model ...] + +Exits with status 1 if any model mismatches or fails to produce a solution. +""" + +from __future__ import annotations + +import argparse +import os +import shutil +import subprocess +import sys +import tempfile +from concurrent.futures import ThreadPoolExecutor +from dataclasses import dataclass + +_DEFAULT_BASELINE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "baseline.txt") + +# Column indices in a GAMS trace record written with traceopt=3. +_TRC_MODEL_STATUS = 13 +_TRC_SOLVER_STATUS = 14 +_TRC_OBJ = 15 +_TRC_SOLVER_TIME = 17 + + +@dataclass +class Result: + name: str + model_type: str + ref: float + obj: float | None = None + model_status: str = "NA" + solver_status: str = "NA" + solver_time: str = "NA" + error: str | None = None + + def rel_diff(self) -> float | None: + if self.obj is None: + return None + return abs(self.obj - self.ref) / max(1.0, abs(self.ref)) + + +def read_baseline(path: str) -> list[tuple[str, str, float]]: + entries = [] + with open(path) as f: + for line in f: + fields = line.split() + if len(fields) == 3: + entries.append((fields[0], fields[1], float(fields[2]))) + return entries + + +def find_gams_dir(gams_dir: str | None) -> str: + if gams_dir is None: + gams = shutil.which("gams") + if gams is None: + sys.exit("error: gams not found in PATH, use --gams-dir") + gams_dir = os.path.dirname(os.path.realpath(gams)) + if not os.path.isfile(os.path.join(gams_dir, "gams")): + sys.exit(f"error: no gams executable in {gams_dir}") + return gams_dir + + +def run_model(gams_dir: str, work_dir: str, name: str, model_type: str, ref: float, + reslim: int, timeout: int) -> Result: + res = Result(name, model_type, ref) + model_dir = os.path.join(work_dir, name) + os.makedirs(model_dir, exist_ok=True) + try: + subprocess.run([os.path.join(gams_dir, "gamslib"), "-q", name], cwd=model_dir, + check=True, capture_output=True) + proc = subprocess.run( + [os.path.join(gams_dir, "gams"), name, "solver=cuopt", "optcr=0", "optca=0", + f"reslim={reslim}", "trace=trc.txt", "traceopt=3", "lo=2"], + cwd=model_dir, capture_output=True, timeout=timeout) + except subprocess.CalledProcessError: + res.error = "gamslib failed" + return res + except subprocess.TimeoutExpired: + res.error = f"timeout after {timeout}s" + return res + + trc_path = os.path.join(model_dir, "trc.txt") + records = [] + if os.path.isfile(trc_path): + with open(trc_path) as f: + records = [l.strip().split(",") for l in f if l.strip() and not l.startswith("*")] + if not records: + res.error = f"no solve recorded (gams rc={proc.returncode})" + return res + + rec = records[-1] + res.model_status = rec[_TRC_MODEL_STATUS] + res.solver_status = rec[_TRC_SOLVER_STATUS] + res.solver_time = rec[_TRC_SOLVER_TIME] + try: + res.obj = float(rec[_TRC_OBJ]) + except ValueError: + res.error = "no objective value" + if proc.returncode != 0 and res.error is None: + res.error = f"gams rc={proc.returncode}" + return res + + +def main() -> int: + parser = argparse.ArgumentParser(description=(__doc__ or "").splitlines()[0]) + parser.add_argument("models", nargs="*", help="subset of models to run (default: all)") + parser.add_argument("-g", "--gams-dir", help="GAMS system directory (default: from PATH)") + parser.add_argument("-b", "--baseline", default=_DEFAULT_BASELINE, help="reference file") + parser.add_argument("-r", "--reslim", type=int, default=120, help="time limit per solve in s") + parser.add_argument("-j", "--jobs", type=int, default=1, help="models to run in parallel") + parser.add_argument("-t", "--rtol", type=float, default=1e-6, + help="tolerance for |obj-ref|/max(1,|ref|)") + parser.add_argument("-k", "--keep", action="store_true", help="keep the work directory") + args = parser.parse_args() + + gams_dir = find_gams_dir(args.gams_dir) + entries = read_baseline(args.baseline) + if args.models: + unknown = set(args.models) - {e[0] for e in entries} + if unknown: + sys.exit(f"error: not in baseline: {' '.join(sorted(unknown))}") + entries = [e for e in entries if e[0] in args.models] + + work_dir = tempfile.mkdtemp(prefix="gamslib-cuopt-") + # Some models set their own resLim, so leave generous headroom beyond --reslim. + timeout = 2 * args.reslim + 60 + + row = "{:6} {:12} {:6} {:>9} {:>9} {:>9} {:>17} {:>17} {}" + print(f"cuOpt vs. CPLEX on {len(entries)} gamslib models " + f"(gams={gams_dir}, reslim={args.reslim}, rtol={args.rtol:g})") + header = row.format("Result", "Model", "Type", "ModelStat", "SolveStat", "Time", + "cuOpt Obj", "CPLEX Obj", "RelDiff") + print(header) + print("-" * len(header), flush=True) + + def job(entry): + name, model_type, ref = entry + res = run_model(gams_dir, work_dir, name, model_type, ref, args.reslim, timeout) + rel = res.rel_diff() + ok = res.error is None and rel is not None and rel <= args.rtol + obj = "NA" if res.obj is None else f"{res.obj:.10g}" + detail = res.error or f"{rel:.2e}" + print(row.format("OK" if ok else "FAIL", res.name, res.model_type, res.model_status, + res.solver_status, res.solver_time, obj, f"{res.ref:.10g}", detail), + flush=True) + return res, ok + + with ThreadPoolExecutor(max_workers=args.jobs) as pool: + results = list(pool.map(job, entries)) + + failed = [r for r, ok in results if not ok] + print(f"\n{len(results) - len(failed)}/{len(results)} models match the CPLEX reference") + if failed: + print("Mismatches / failures:") + for r in failed: + rel = r.rel_diff() + detail = r.error or f"cuopt={r.obj:.10g} cplex={r.ref:.10g} rel={rel:.2e}" + print(f" {r.name:12} ms={r.model_status:3} {detail}") + + if args.keep or failed: + print(f"\nWork directory: {work_dir}") + else: + shutil.rmtree(work_dir, ignore_errors=True) + return 1 if failed else 0 + + +if __name__ == "__main__": + sys.exit(main())