From ebca03916022479034e0250a76ee8ea98cc6bea4 Mon Sep 17 00:00:00 2001
From: shimwell <8583900+shimwell@users.noreply.github.com>
Date: Tue, 20 Oct 2026 12:04:20 +0200
Subject: [PATCH 1/5] Unify JEFF ENDF generation for 3.3 and 4.0
---
.github/workflows/test_package.yml | 1 +
README.md | 82 ++++
pyproject.toml | 1 +
src/openmc_data/generate/generate_jeff.py | 264 ++++++++++
src/openmc_data/generate/generate_jeff33.py | 212 +-------
src/openmc_data/urls_jeff.py | 151 ++++++
src/openmc_data/utils.py | 26 +-
tests/test_generate_jeff.py | 504 ++++++++++++++++++++
tests/test_urls.py | 58 +++
tests/test_utils.py | 114 +++++
10 files changed, 1200 insertions(+), 213 deletions(-)
create mode 100644 src/openmc_data/generate/generate_jeff.py
create mode 100644 src/openmc_data/urls_jeff.py
create mode 100644 tests/test_generate_jeff.py
diff --git a/.github/workflows/test_package.yml b/.github/workflows/test_package.yml
index 9fbbfa5..27c9848 100644
--- a/.github/workflows/test_package.yml
+++ b/.github/workflows/test_package.yml
@@ -63,6 +63,7 @@ jobs:
run: |
generate_cendl --help
generate_endf --help
+ generate_jeff --help
generate_jeff33 --help
generate_jendl --help
diff --git a/README.md b/README.md
index 609d98d..3b70898 100644
--- a/README.md
+++ b/README.md
@@ -101,8 +101,90 @@ A few categories of scripts are available:
| generate_endf | ENDF/B | VII.1
VIII.0
VIII.1 | NNDC |
| generate_fendl | FENDL | 3.2c
3.2b
3.2a
3.2
3.1d
3.1a
3.0 | |
| generate_jendl | JENDL | 4.0
5.0 | |
+| generate_jeff | JEFF | 3.3
4.0 | NJOY |
+| generate_jeff33 | JEFF | 3.3 (legacy alias) | NJOY |
| generate_tendl | TENDL | 2023
2025 | |
+#### JEFF generation
+
+`generate_jeff` processes ENDF evaluations with OpenMC and NJOY, defaulting to
+JEFF 4.0. Use `--release 3.3` for JEFF 3.3, and `--particles neutron thermal
+photon` to select particle types (all three are selected by default).
+Thermal-only generation downloads the neutron evaluations it needs, but does
+not produce neutron HDF5 files. `--jobs` controls parallel workers and defaults
+to the CPU count; `--jobs 1` processes serially.
+
+For example, `generate_jeff --release 4.0 --particles neutron --temperatures
+293.6 600 --jobs 4` generates neutron data at two temperatures. The default
+neutron temperatures are 250, 293.6, 600, 900, 1200 and 2500 K. Thermal
+scattering always uses the evaluation's native temperatures, regardless of
+`--temperatures`. NJOY must be on `PATH`. `--libver earliest` (default) and
+`--libver latest` control HDF5 compatibility.
+
+Downloads come from the NEA records for
+[JEFF 3.3 neutrons](https://data.oecd-nea.org/records/bh7jn-rm903),
+[JEFF 3.3 thermal scattering](https://data.oecd-nea.org/records/3nm84-46r21),
+[JEFF 4.0 neutrons](https://data.oecd-nea.org/records/e9ajn-a3p20), and
+[JEFF 4.0 thermal scattering](https://data.oecd-nea.org/records/scmva-nqh68).
+Published MD5 checksums are checked, including when extracting a local cache.
+Photon data are supplements from ENDF/B-VIII.0 for JEFF 3.3 and ENDF/B-VIII.1
+for JEFF 4.0, not JEFF evaluations.
+
+JEFF 3.3 C13 and O17 neutron evaluations are replaced with the individual
+[TENDL-2019 archives published by IAEA](https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/).
+The original JEFF files remain untouched. Each replacement ZIP must contain
+one evaluation with the expected ENDF target. IAEA does not publish checksums
+for these ZIPs; extraction checks ZIP CRCs. Some networks receive a Cloudflare
+challenge from IAEA. This is a download failure, not a successful generation;
+in that case, obtain the archives manually and reuse a local cache.
+
+JEFF 3.3 retains the existing 20 thermal pairs. JEFF 4.0 uses the 128 thermal
+pairs in the [upstream generator](https://github.com/openmc-dev/data/blob/master/generate_jeff.py),
+checked against NEA's archive inventories. Additional temperature-split
+PuO2/UO2/Zy4 evaluations in the JEFF 4.0 archive are not included. Availability
+in an archive does not guarantee that every evaluation is processable by a
+particular OpenMC/NJOY version; full-library NJOY qualification is not part of
+the automated tests.
+
+The default destination is `jeff-RELEASE-hdf5`, with particle subdirectories
+and a relative-path XML library manifest. `--destination` overrides it. Each
+worker writes to isolated staging storage; missing inputs, duplicate names,
+unpaired photon inputs, or worker errors abort generation. Only outputs from
+the current run are registered, never stale files already in the destination.
+Processing or validation failures leave an existing library unchanged. During
+publication the old manifest is removed before replacing data, and the new
+manifest is installed last, so a publication failure cannot advertise mixed
+data as a successful library.
+
+Archive and extracted-input directories default to `jeff-RELEASE-download`
+and `jeff-RELEASE-endf`. Override these with `--download-dir` and `--endf-dir`.
+Both use `neutron`, `thermal`, and `photon` subdirectories. NEA archives retain
+their original basenames despite the download URLs ending in `/content`.
+JEFF 3.3 replacement archives and extracted files go into separate
+`replacements/6-C-13g` and `replacements/8-O-17g` subdirectories of those roots.
+The output and two cache roots must not overlap.
+
+- `--no-download`: extract cached archives without accessing the network.
+- `--no-download --no-extract`: process local inputs in the particle
+ subdirectories, including nested archive folders. Neutron/photon subsets are
+ allowed; thermal generation requires all mapped pairs for the release.
+- `--cleanup`: after success, remove only the selected particle/replacement
+ cache directories for enabled download/extraction stages. Offline inputs are
+ retained when both stages are disabled.
+- `--use-tmpdir`: use automatically cleaned temporary inputs for a fresh
+ download and extraction. When reusing inputs or specifying cache roots, this
+ option is ignored with a warning instead of losing access to local files.
+
+`generate_jeff33` remains an import-safe compatibility wrapper. It fixes the
+release to 3.3 and keeps its legacy destination `jeff33_hdf5` and temporary-input
+default. Reusable inputs use the new release-scoped directory layout described
+above, not the legacy working-directory layout.
+
+Offline orchestration tests use real HDF5/XML with mocked NJOY. Separate live
+source tests check archive availability, NEA checksums, and the 4.0 inventory.
+Opt-in NJOY smoke tests use bounded streamed samples rather than downloading
+complete libraries; set `OPENMC_JEFF_INTEGRATION=1` with NJOY on `PATH` when
+running the integration tests.
### Download cross sections
diff --git a/pyproject.toml b/pyproject.toml
index 942fab3..e1bc05b 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -71,6 +71,7 @@ convert_tendl = "openmc_data.convert.convert_tendl:main"
generate_cendl = "openmc_data.generate.generate_cendl:main"
generate_endf = "openmc_data.generate.generate_endf:main"
+generate_jeff = "openmc_data.generate.generate_jeff:main"
generate_jeff33 = "openmc_data.generate.generate_jeff33:main"
generate_jendl = "openmc_data.generate.generate_jendl:main"
generate_fendl = "openmc_data.generate.generate_fendl:main"
diff --git a/src/openmc_data/generate/generate_jeff.py b/src/openmc_data/generate/generate_jeff.py
new file mode 100644
index 0000000..95ca00a
--- /dev/null
+++ b/src/openmc_data/generate/generate_jeff.py
@@ -0,0 +1,264 @@
+#!/usr/bin/env python3
+"""Generate OpenMC HDF5 data from JEFF 3.3 or 4.0 ENDF evaluations using NJOY.
+
+Photons use ENDF/B-VIII.0 (JEFF 3.3) or VIII.1 (JEFF 4.0).
+Thermal scattering uses the native evaluation temperatures, not --temperatures.
+"""
+
+import argparse
+from contextlib import ExitStack
+import math
+from multiprocessing import Pool
+from pathlib import Path
+import shutil
+from tempfile import TemporaryDirectory
+import warnings
+
+from openmc_data import download, extract, process_neutron, process_thermal, ProgressTracker
+from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements
+from openmc_data.utils import _file_md5
+
+
+def _parser(legacy=False):
+ parser = argparse.ArgumentParser(description=__doc__,
+ formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+ parser.add_argument('-r', '--release', choices=['3.3'] if legacy else ['3.3', '4.0'],
+ default='3.3' if legacy else '4.0')
+ parser.add_argument('-d', '--destination', type=Path,
+ default=Path('jeff33_hdf5') if legacy else None)
+ parser.add_argument('-p', '--particles', nargs='+', choices=['neutron', 'thermal', 'photon'],
+ default=['neutron', 'thermal', 'photon'])
+ parser.add_argument('--temperatures', nargs='+', type=float,
+ default=[250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0],
+ help='Neutron temperatures in kelvin (thermal uses native temperatures)')
+ parser.add_argument('--jobs', type=int, default=None, help='NJOY worker count; default: CPU count')
+ parser.add_argument('--libver', choices=['earliest', 'latest'], default='earliest')
+ parser.add_argument('--download-dir', type=Path, help='Archive cache (default: jeff-RELEASE-download)')
+ parser.add_argument('--endf-dir', type=Path, help='Extracted inputs (default: jeff-RELEASE-endf)')
+ for flag, default, help_text in [
+ ('download', True, 'Download missing or invalid archives'),
+ ('extract', True, 'Extract archives into particle subdirectories'),
+ ('cleanup', False, 'Remove selected caches after success, only for enabled download/extract stages'),
+ ('use-tmpdir', legacy, 'Use temporary inputs for a fresh download and extraction'),
+ ]:
+ parser.add_argument('--' + flag, dest=flag.replace('-', '_'), action='store_true', help=help_text)
+ parser.add_argument('--no-' + flag, dest=flag.replace('-', '_'), action='store_false')
+ parser.set_defaults(**{flag.replace('-', '_'): default})
+ return parser
+
+
+def _prepare(details, archives, inputs, args):
+ """Keep caches reusable offline and check checksums even with --no-download."""
+ for name, checksum in zip(details['compressed_files'], details['checksums']):
+ archive = archives / Path(name).name
+ if args.download and not (archive.is_file() and checksum
+ and _file_md5(archive) == checksum):
+ download(details['base_url'] + name + details.get('url_suffix', ''),
+ checksum=checksum, output_path=archives,
+ output_filename=archive.name, timeout=60)
+ if args.extract:
+ if not archive.is_file():
+ raise FileNotFoundError(f'Missing archive {archive}; enable --download or supply the cache')
+ if checksum and _file_md5(archive) != checksum:
+ raise OSError(f'MD5 checksum does not match: {archive}')
+ if args.extract:
+ # A fresh extraction avoids discovering stale evaluations left by an old archive.
+ inputs.parent.mkdir(parents=True, exist_ok=True)
+ with TemporaryDirectory(prefix='.jeff-extract-', dir=inputs.parent) as tmp:
+ staging = Path(tmp) / 'inputs'
+ for name in details['compressed_files']:
+ extract(archives / Path(name).name, staging)
+ if inputs.exists():
+ shutil.rmtree(inputs)
+ staging.replace(inputs)
+
+
+def _index(directory, pattern):
+ result = {}
+ for path in sorted(directory.rglob(pattern)):
+ if path.is_file():
+ if path.name in result:
+ raise ValueError(f'Ambiguous input {path.name}: {result[path.name]} and {path}')
+ result[path.name] = path
+ if not result:
+ raise FileNotFoundError(f'No {pattern} inputs in {directory}; enable --extract or supply ENDF files')
+ return result
+
+
+def _required(index, name):
+ if name not in index:
+ raise FileNotFoundError(f'Missing required evaluation: {name}')
+ return index[name]
+
+
+def _replacement(details, directory):
+ import openmc.data
+
+ # IAEA ZIPs contain one ENDF evaluation; do not assume its member filename.
+ paths = [p for p in directory.rglob('*') if p.is_file()]
+ if len(paths) != 1:
+ raise ValueError(f'Expected one TENDL replacement evaluation in {directory}, found {len(paths)}')
+ evaluation = openmc.data.endf.Evaluation(paths[0])
+ target = evaluation.target
+ if (target['atomic_number'], target['mass_number']) != details['target'] or target['isomeric_state']:
+ raise ValueError(f'Wrong TENDL replacement target in {paths[0]}')
+ return paths[0]
+
+
+def _photon_pairs(directory):
+ photo = {name[len('photoat-'):]: path for name, path in _index(directory, 'photoat-*.endf').items()}
+ atom = {name[len('atom-'):]: path for name, path in _index(directory, 'atom-*.endf').items()}
+ if photo.keys() != atom.keys():
+ raise ValueError(f'Unpaired photon evaluations: {sorted(photo.keys() ^ atom.keys())}')
+ return [(photo[key], atom[key]) for key in sorted(photo)]
+
+
+def _process_task(particle, paths, output, libver, temperatures):
+ """Each task has its own output directory, so name collisions cannot overwrite data."""
+ output.mkdir(parents=True)
+ try:
+ if particle == 'neutron':
+ process_neutron(paths[0], output, libver, temperatures)
+ elif particle == 'thermal':
+ process_thermal(*paths, output, libver)
+ else:
+ import openmc.data
+ data = openmc.data.IncidentPhoton.from_endf(*paths)
+ data.export_to_hdf5(output / f'{data.name}.h5', 'w', libver=libver)
+ files = list(output.glob('*.h5'))
+ if len(files) != 1 or not files[0].stat().st_size:
+ raise RuntimeError(f'Expected one nonempty HDF5 output in {output}')
+ return particle, files[0]
+ except Exception as exc:
+ raise RuntimeError(f'Failed processing {particle}: {paths}') from exc
+
+
+def _generate(tasks, destination, args):
+ import openmc.data
+
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ with TemporaryDirectory(prefix='.jeff-generate-', dir=destination.parent) as tmp:
+ staging = Path(tmp)
+ work = [(particle, paths, staging / 'jobs' / str(i), args.libver, args.temperatures)
+ for i, (particle, paths) in enumerate(tasks)]
+ tracker = ProgressTracker(len(work), verb='Processed')
+ if args.jobs == 1:
+ results = []
+ for task in work:
+ results.append(_process_task(*task))
+ tracker.complete(task[1][-1].name)
+ else:
+ with Pool(processes=args.jobs) as pool:
+ pending = [pool.apply_async(_process_task, task,
+ callback=tracker.callback(task[1][-1].name)) for task in work]
+ # get(), not wait(): any worker error aborts before publication.
+ results = [result.get() for result in pending]
+
+ library = openmc.data.DataLibrary()
+ outputs = []
+ seen = set()
+ for particle, path in sorted(results, key=lambda item: (item[0], item[1].name)):
+ relative = Path(particle) / path.name
+ if relative in seen:
+ raise ValueError(f'Output name collision: {relative}')
+ seen.add(relative)
+ target = staging / 'library' / relative
+ target.parent.mkdir(parents=True, exist_ok=True)
+ path.replace(target)
+ library.register_file(target)
+ outputs.append(relative)
+ manifest = staging / 'library' / 'cross_sections.xml'
+ library.export_to_xml(manifest)
+
+ # Make paths relative to the manifest, not the temporary staging directory.
+ # DataLibrary may otherwise add an absolute for single-particle runs.
+ from xml.etree import ElementTree as ET
+ tree = ET.parse(manifest)
+ root = tree.getroot()
+ directory = root.find('directory')
+ if directory is not None:
+ root.remove(directory)
+ for entry, relative in zip(root.findall('library'), outputs):
+ entry.set('path', relative.as_posix())
+ tree.write(manifest, encoding='utf-8', xml_declaration=True)
+
+ destination.mkdir(parents=True, exist_ok=True)
+ # Old data remain untouched on processing/validation failure. Once publishing
+ # starts, remove the old manifest so an I/O failure cannot advertise mixed data.
+ (destination / 'cross_sections.xml').unlink(missing_ok=True)
+ for relative in outputs:
+ target = destination / relative
+ target.parent.mkdir(parents=True, exist_ok=True)
+ (staging / 'library' / relative).replace(target)
+ manifest.replace(destination / 'cross_sections.xml')
+
+
+def _main(argv=None, *, legacy=False):
+ parser = _parser(legacy)
+ args = parser.parse_args(argv)
+ if args.jobs is not None and args.jobs < 1:
+ parser.error('--jobs must be positive')
+ if any(not math.isfinite(t) or t <= 0 for t in args.temperatures):
+ parser.error('--temperatures must be finite and positive')
+ if len({round(t) for t in args.temperatures}) != len(args.temperatures):
+ parser.error('--temperatures must have distinct rounded kelvin values (OpenMC HDF5 labels)')
+ particles = set(args.particles)
+ required = [p for p in ('neutron', 'thermal', 'photon')
+ if p in particles or (p == 'neutron' and 'thermal' in particles)]
+ details = jeff_endf_details[args.release]
+ destination = (args.destination or Path(f'jeff-{args.release}-hdf5')).resolve()
+ archives = (args.download_dir or Path(f'jeff-{args.release}-download')).resolve()
+ inputs = (args.endf_dir or Path(f'jeff-{args.release}-endf')).resolve()
+ for a, b in ((destination, archives), (destination, inputs), (archives, inputs)):
+ if a == b or a in b.parents or b in a.parents:
+ parser.error('Destination, download and ENDF directories must not overlap')
+
+ with ExitStack() as stack:
+ if args.use_tmpdir:
+ if args.download and args.extract and not args.download_dir and not args.endf_dir:
+ temporary = Path(stack.enter_context(TemporaryDirectory(prefix='jeff-inputs-')))
+ archives, inputs = temporary / 'download', temporary / 'endf'
+ else:
+ warnings.warn('--use-tmpdir ignored when reusing local inputs or explicit cache directories')
+ prepared = []
+ for particle in required:
+ _prepare(details[particle], archives / particle, inputs / particle, args)
+ prepared.append(particle)
+
+ tasks = []
+ if 'neutron' in required:
+ neutrons = _index(inputs / 'neutron', details['neutron']['pattern'])
+ if 'neutron' in particles:
+ selected = dict(neutrons)
+ if args.release == '3.3':
+ for name, replacement in jeff33_replacements.items():
+ if name in selected:
+ group = 'replacements/' + Path(name).stem
+ _prepare(replacement, archives / group, inputs / group, args)
+ prepared.append(group)
+ selected[name] = _replacement(replacement, inputs / group)
+ print(f'Using TENDL-2019 replacement for {name}')
+ tasks.extend(('neutron', (path,)) for path in selected.values())
+ if 'thermal' in particles:
+ thermal = _index(inputs / 'thermal', details['neutron']['pattern'])
+ tasks.extend(('thermal', (_required(neutrons, n), _required(thermal, t)))
+ for n, t in details['thermal']['pairs'])
+ if 'photon' in particles:
+ tasks.extend(('photon', pair) for pair in _photon_pairs(inputs / 'photon'))
+ if not tasks:
+ raise ValueError('No evaluations selected for generation')
+ _generate(tasks, destination, args)
+ if args.cleanup:
+ for group in prepared:
+ for root, enabled in ((archives, args.download), (inputs, args.extract)):
+ if enabled and (root / group).exists():
+ shutil.rmtree(root / group)
+ print(f'Wrote {destination / "cross_sections.xml"}')
+
+
+def main(argv=None):
+ _main(argv)
+
+
+if __name__ == '__main__':
+ main()
\ No newline at end of file
diff --git a/src/openmc_data/generate/generate_jeff33.py b/src/openmc_data/generate/generate_jeff33.py
index 5828c5c..f380d47 100755
--- a/src/openmc_data/generate/generate_jeff33.py
+++ b/src/openmc_data/generate/generate_jeff33.py
@@ -1,215 +1,11 @@
#!/usr/bin/env python3
+"""Legacy JEFF 3.3 entry point; implementation lives in generate_jeff."""
-"""
-Generate JEFF 3.3 HDF5 library for use in OpenMC by first processing ENDF files
-using NJOY. The resulting library will contain incident neutron, photoatomic,
-and thermal scattering data. Note that JEFF doesn't distribute photoatomic or
-atomic relaxation data so these are obtained from ENDF/B-VIII.0.
+from .generate_jeff import _main
-"""
-import argparse
-import os
-import sys
-import tarfile
-import tempfile
-import zipfile
-from multiprocessing import Pool
-from pathlib import Path
-from urllib.parse import urljoin
-
-import openmc.data
-
-from openmc_data import download, process_neutron, process_thermal, ProgressTracker
-
-# Make sure Python version is sufficient
-assert sys.version_info >= (3, 6), "Python 3.6+ is required"
-
-
-class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
- argparse.RawDescriptionHelpFormatter):
- pass
-
-
-parser = argparse.ArgumentParser(
- description=__doc__,
- formatter_class=CustomFormatter
-)
-parser.add_argument('-d', '--destination', type=Path, default=Path('jeff33_hdf5'),
- help='Directory to create new library in')
-parser.add_argument('--libver', choices=['earliest', 'latest'],
- default='earliest', help="Output HDF5 versioning. Use "
- "'earliest' for backwards compatibility or 'latest' for "
- "performance")
-parser.add_argument('--download', action='store_true',
- help='Download files')
-parser.add_argument('--no-download', dest='download', action='store_false',
- help='Do not download files')
-parser.add_argument('--use-tmpdir', dest='tmpdir', action='store_true',
- help='Use temporary directory while processing')
-parser.add_argument('--no-use-tmpdir', dest='tmpdir', action='store_false',
- help='Do not use temporary directory while processing')
-parser.add_argument('--extract', action='store_true',
- help='Extract zip files')
-parser.add_argument('--no-extract', dest='extract', action='store_false',
- help='Do not extract zip files')
-parser.add_argument('--temperatures', type=float,
- default=[250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0],
- help="Temperatures in Kelvin", nargs='+')
-parser.set_defaults(download=True, extract=True, tmpdir=True)
-args = parser.parse_args()
-
-
-def sort_key(path):
- if path.name.startswith('c_'):
- # Ensure that thermal scattering gets sorted after neutron data
- return (1000, path)
- else:
- return openmc.data.zam(path.stem)
-
-
-def main():
-
- base_endf = 'https://www.nndc.bnl.gov/endf-b8.0/zips/'
- base_jeff = 'http://www.oecd-nea.org/dbdata/jeff/jeff33/downloads/'
- files = [
- (base_jeff, 'JEFF33-n.tgz', 'e540bbf95179257280c61acfa75c83de'),
- (base_jeff, 'JEFF33-tsl.tgz', '82a6df4cb802aa4a09b95309f7861c54'),
- (base_endf, 'ENDF-B-VIII.0_photoat.zip', 'd49f5b54be278862e1ce742ccd94f5c0'),
- (base_endf, 'ENDF-B-VIII.0_atomic_relax.zip', 'e04d50098cb2a7e4fe404ec4071611cc'),
- ]
-
- # TODO: these individual TENDL neutron file endpoints return 404 on the
- # new tendl.imperial.ac.uk site. Re-enable once the files are available.
- tendl_files = [
- # 'https://tendl.imperial.ac.uk/tendl_2019/neutron_file/C/C013/lib/endf/n-C013.tendl',
- # 'https://tendl.imperial.ac.uk/tendl_2019/neutron_file/O/O017/lib/endf/n-O017.tendl',
- ]
-
- neutron_dir = Path('endf6')
- thermal_dir = Path('JEFF33-tsl')
- photoat_dir = Path('ENDF-B-VIII.0_photoat')
- atomic_dir = Path('ENDF-B-VIII.0_atomic_relax')
- thermal_paths = [
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinCaH2.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinCH2.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinH2O.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinIce.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinMesitylene-PhaseII.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinOrthoH.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinParaH.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinToluene.jeff33'),
- (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinZrH.jeff33'),
- (neutron_dir / '1-H-2g.jeff33', thermal_dir / 'tsl-DinD2O.jeff33'),
- (neutron_dir / '1-H-2g.jeff33', thermal_dir / 'tsl-DinOrthoD.jeff33'),
- (neutron_dir / '1-H-2g.jeff33', thermal_dir / 'tsl-DinParaD.jeff33'),
- (neutron_dir / '4-Be-9g.jeff33', thermal_dir / 'tsl-Be.jeff33'),
- (neutron_dir / '6-C-0g.jeff33', thermal_dir / 'tsl-Graphite.jeff33'),
- (neutron_dir / '8-O-16g.jeff33', thermal_dir / 'tsl-O16inAl2O3.jeff33'),
- (neutron_dir / '8-O-16g.jeff33', thermal_dir / 'tsl-OinD2O.jeff33'),
- (neutron_dir / '12-Mg-24g.jeff33', thermal_dir / 'tsl-Mg.jeff33'),
- (neutron_dir / '13-Al-27g.jeff33', thermal_dir / 'tsl-Al27inAl2O3.jeff33'),
- (neutron_dir / '14-Si-28g.jeff33', thermal_dir / 'tsl-Silicon.jeff33'),
- (neutron_dir / '20-Ca-40g.jeff33', thermal_dir / 'tsl-CainCaH2.jeff33'),
- ]
-
- pwd = Path.cwd()
-
- destination = args.destination.resolve()
- (destination / 'photon').mkdir(parents=True, exist_ok=True)
-
- with tempfile.TemporaryDirectory() as tmpdir:
- # Save current working directory and temporarily change dir
- if args.tmpdir:
- os.chdir(tmpdir)
- library = openmc.data.DataLibrary()
-
- # =========================================================================
- # DOWNLOAD FILES
- if args.download:
- for base, fname, checksum in files:
- download(urljoin(base, fname), checksum)
-
- # =========================================================================
- # EXTRACT ARCHIVES
-
- if args.extract:
- for _, f, _ in files:
- if f.endswith('.zip'):
- print(f'Extracting {f}...')
- with zipfile.ZipFile(f) as zipf:
- zipf.extractall()
- elif f.endswith('.tar.gz') or f.endswith('.tgz'):
- print(f'Extracting {f}...')
- with tarfile.open(f) as tgz:
- tgz.extractall()
-
- # =========================================================================
- # REPLACE C13 AND O17 WITH FILES FROM TENDL 2019
-
- # The evaluations for C13 and O17 can't be processed by recent versions of
- # NJOY 2016 because of a bug in MF=6 in the evaluations. Instead of trying
- # to patch the evaluation, we just replace it with more recent evaluations
- # from TENDL 2019
-
- for url in tendl_files:
- download(url, output_path=neutron_dir)
-
- (neutron_dir / 'n-C013.tendl').rename(neutron_dir / '6-C-13g.jeff33')
- (neutron_dir / 'n-O017.tendl').rename(neutron_dir / '8-O-17g.jeff33')
-
- # =========================================================================
- # PROCESS INCIDENT NEUTRON AND THERMAL SCATTERING DATA IN PARALLEL
-
- with Pool() as pool:
- neutron_paths = sorted(neutron_dir.glob('*.jeff33'))
- results = []
- tracker = ProgressTracker(
- len(neutron_paths) + len(thermal_paths), verb='Processed')
- for p in neutron_paths:
- func_args = (p, destination, args.libver, args.temperatures)
- r = pool.apply_async(
- process_neutron, func_args,
- callback=tracker.callback(p.name),
- error_callback=tracker.callback(p.name),
- )
- results.append(r)
- for p_neut, p_therm in thermal_paths:
- func_args = (p_neut, p_therm, destination, args.libver)
- r = pool.apply_async(
- process_thermal, func_args,
- callback=tracker.callback(p_therm.name),
- error_callback=tracker.callback(p_therm.name),
- )
- results.append(r)
- for r in results:
- r.wait()
-
- for p in sorted(destination.glob('*.h5'), key=sort_key):
- library.register_file(p)
-
- # =========================================================================
- # INCIDENT PHOTON DATA
-
- for z in range(1, 101):
- element = openmc.data.ATOMIC_SYMBOL[z]
- print('Generating HDF5 file for Z={} ({})...'.format(z, element))
-
- # Generate instance of IncidentPhoton
- photo_file = photoat_dir / f'photoat-{z:03}_{element}_000.endf'
- atom_file = atomic_dir / f'atom-{z:03}_{element}_000.endf'
- data = openmc.data.IncidentPhoton.from_endf(photo_file, atom_file)
-
- # Write HDF5 file and register it
- outfile = destination / 'photon' / f'{element}.h5'
- data.export_to_hdf5(outfile, 'w', libver=args.libver)
- library.register_file(outfile)
-
- library.export_to_xml(destination / 'cross_sections.xml')
-
- # Change back to original directory
- if args.tmpdir:
- os.chdir(pwd)
+def main(argv=None):
+ _main(argv, legacy=True)
if __name__ == '__main__':
diff --git a/src/openmc_data/urls_jeff.py b/src/openmc_data/urls_jeff.py
new file mode 100644
index 0000000..f0ab666
--- /dev/null
+++ b/src/openmc_data/urls_jeff.py
@@ -0,0 +1,151 @@
+"""ENDF inputs for JEFF generation, separate from the processed ACE registry.
+
+NEA record metadata and archive previews were checked on 2026-10-20.
+The JEFF 4.0 pairs follow openmc-dev/data's generate_jeff.py (128 pairs).
+In particular, ``tsl_O_D20.jeff`` really uses a zero in the archive name.
+The additional temperature-split PuO2/UO2/Zy4 evaluations are not mapped by
+that upstream generator and are deliberately not included here.
+"""
+
+
+THERMAL_PAIRS_33 = (
+ ('1-H-1g.jeff33', 'tsl-HinCaH2.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinCH2.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinH2O.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinIce.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinMesitylene-PhaseII.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinOrthoH.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinParaH.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinToluene.jeff33'),
+ ('1-H-1g.jeff33', 'tsl-HinZrH.jeff33'),
+ ('1-H-2g.jeff33', 'tsl-DinD2O.jeff33'),
+ ('1-H-2g.jeff33', 'tsl-DinOrthoD.jeff33'),
+ ('1-H-2g.jeff33', 'tsl-DinParaD.jeff33'),
+ ('4-Be-9g.jeff33', 'tsl-Be.jeff33'),
+ ('6-C-0g.jeff33', 'tsl-Graphite.jeff33'),
+ ('8-O-16g.jeff33', 'tsl-O16inAl2O3.jeff33'),
+ ('8-O-16g.jeff33', 'tsl-OinD2O.jeff33'),
+ ('12-Mg-24g.jeff33', 'tsl-Mg.jeff33'),
+ ('13-Al-27g.jeff33', 'tsl-Al27inAl2O3.jeff33'),
+ ('14-Si-28g.jeff33', 'tsl-Silicon.jeff33'),
+ ('20-Ca-40g.jeff33', 'tsl-CainCaH2.jeff33'),
+)
+
+# Grouped by incident evaluation; material suffixes retain the archive spelling.
+_THERMAL_MATERIALS_40 = {
+ '1-H-001g': '''H_Benzene H_C5O2H8 H_C8H8 H_CaH2 H_CaO2H2 H_CH2
+ H_CH4-liquid H_CH4-solid H_H2O H_HF H_Ice H_KOH H_LiH H_Mesitylene
+ H_MgH2 H_MgOH2 H_NaMgH3 H_NaOH H_ortho-H H_para-H H_ParaffinicOil
+ H_SrH2 H_Toluene H_UH3 H_YH2 H_ZrH H_ZrH15 H_ZrH2''',
+ '1-H-002g': 'D_7LiD D_D2O D_MgD2 D_ortho-D D_para-D',
+ '3-Li-007g': '7Li_7LiD Li_FLiBe Li_LiF Li_LiH',
+ '4-Be-009g': 'Be_Be Be_Be2C Be_BeF2 Be_BeO Be_FLiBe',
+ '6-C-012g': 'C_Be2C C_C5O2H8 C_C8H8 C_CF2 C_CH2 C_Diamond C_Graphite C_UC C_ZrC',
+ '7-N-014g': 'N_GaN N_UN',
+ '8-O-016g': '''O_Al2O3 O_BeO O_C5O2H8 O_CaO2H2 O_D20 O_Ge3Bi4O12
+ O_KOH O_MgO O_MgOH2 O_NaOH O_Y3Al5O12''',
+ '9-F-019g': 'F_BeF2 F_CF2 F_FLiBe F_LiF F_MgF2',
+ '11-Na-023g': 'Na_Na Na_NaI Na_NaMgH3 Na_NaOH',
+ '12-Mg-024g': 'Mg_Mg Mg_MgD2 Mg_MgF2 Mg_MgH2 Mg_MgO Mg_MgOH2 Mg_NaMgH3',
+ '13-Al-027g': 'Al_Al Al_Al2O3 Al_Y3Al5O12',
+ '14-Si-028g': 'Si_Si',
+ '16-S-032g': 'S_ZnS',
+ '19-K-039g': 'K_K K_KOH',
+ '20-Ca-040g': 'Ca_Ca Ca_CaH2 Ca_CaO2H2',
+ '22-Ti-048g': 'Ti_Ti',
+ '23-V-051g': 'V_V',
+ '24-Cr-052g': 'Cr_Cr',
+ '26-Fe-056g': 'Fe_Fe-alpha Fe_Fe-gamma',
+ '28-Ni-058g': 'Ni_Ni',
+ '29-Cu-063g': 'Cu_Cu',
+ '30-Zn-064g': 'Zn_Zn Zn_ZnS',
+ '31-Ga-069g': 'Ga_GaN Ga_GaSe',
+ '32-Ge-074g': 'Ge_Ge Ge_Ge3Bi4O12 Ge_GeTe',
+ '34-Se-080g': 'Se_GaSe',
+ '38-Sr-088g': 'Sr_SrH2',
+ '39-Y-089g': 'Y_Y3Al5O12 Y_YH2',
+ '40-Zr-090g': 'Zr_Zr Zr_ZrC Zr_ZrH15 Zr_ZrH2',
+ '41-Nb-093g': 'Nb_Nb',
+ '42-Mo-098g': 'Mo_Mo',
+ '46-Pd-106g': 'Pd_Pd',
+ '47-Ag-107g': 'Ag_Ag',
+ '50-Sn-120g': 'Sn_Sn',
+ '52-Te-130g': 'Te_GeTe',
+ '53-I-127g': 'I_NaI',
+ '74-W-184g': 'W_W',
+ '78-Pt-195g': 'Pt_Pt',
+ '79-Au-197g': 'Au_Au',
+ '82-Pb-208g': 'Pb_Pb',
+ '83-Bi-209g': 'Bi_Bi Bi_Ge3Bi4O12',
+ '92-U-238g': 'U_U U_UC U_UN',
+}
+THERMAL_PAIRS_40 = tuple(
+ (f'n_{neutron}.jeff', f'tsl_{material}.jeff')
+ for neutron, materials in _THERMAL_MATERIALS_40.items()
+ for material in materials.split()
+)
+
+
+jeff_endf_details = {
+ '3.3': {
+ 'neutron': {
+ 'base_url': 'https://data.oecd-nea.org/api/records/bh7jn-rm903/files/',
+ 'compressed_files': ['JEFF33-n.tgz'],
+ 'url_suffix': '/content',
+ 'checksums': ['e540bbf95179257280c61acfa75c83de'],
+ 'pattern': '*.jeff33',
+ },
+ 'thermal': {
+ 'base_url': 'https://data.oecd-nea.org/api/records/3nm84-46r21/files/',
+ 'compressed_files': ['JEFF33-tsl.tgz'],
+ 'url_suffix': '/content',
+ 'checksums': ['82a6df4cb802aa4a09b95309f7861c54'],
+ 'pairs': THERMAL_PAIRS_33,
+ },
+ 'photon': {
+ 'base_url': 'https://www.nndc.bnl.gov/endf-b8.0/zips/',
+ 'compressed_files': ['ENDF-B-VIII.0_photoat.zip', 'ENDF-B-VIII.0_atomic_relax.zip'],
+ 'checksums': ['d49f5b54be278862e1ce742ccd94f5c0', 'e04d50098cb2a7e4fe404ec4071611cc'],
+ },
+ },
+ '4.0': {
+ 'neutron': {
+ 'base_url': 'https://data.oecd-nea.org/api/records/e9ajn-a3p20/files/',
+ 'compressed_files': ['JEFF40-Evaluations-Neutron-593.zip'],
+ 'url_suffix': '/content',
+ 'checksums': ['51d00ee7bf1491d428f9b30a9782e41d'],
+ 'pattern': '*.jeff',
+ },
+ 'thermal': {
+ 'base_url': 'https://data.oecd-nea.org/api/records/scmva-nqh68/files/',
+ 'compressed_files': ['JEFF40-Evaluations-TSL.zip'],
+ 'url_suffix': '/content',
+ 'checksums': ['13d22cd59f83368885ab2d86300b470a'],
+ 'pairs': THERMAL_PAIRS_40,
+ },
+ 'photon': {
+ 'base_url': 'https://www.nndc.bnl.gov/endf-releases/releases/B-VIII.1/',
+ 'compressed_files': ['photoat/photoat-version.VIII.1.tar.gz',
+ 'atomic_relax/atomic_relax-version.VIII.1.tar.gz'],
+ 'checksums': ['6d5f4830f6290d6c618803a8391ba0cf', '70e9ca0c481236499b7a3e0a490f4ef2'],
+ },
+ },
+}
+
+# Individual IAEA archives avoid downloading the 3 GB official TENDL bulk tar.
+# These replace defective JEFF 3.3 MF=6 evaluations, without modifying originals.
+# IAEA does not publish checksums here; ZIP CRCs and the ENDF target are checked.
+jeff33_replacements = {
+ '6-C-13g.jeff33': {
+ 'base_url': 'https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/',
+ 'compressed_files': ['n_6-C-13_0628.zip'],
+ 'checksums': [None],
+ 'target': (6, 13),
+ },
+ '8-O-17g.jeff33': {
+ 'base_url': 'https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/',
+ 'compressed_files': ['n_8-O-17_0828.zip'],
+ 'checksums': [None],
+ 'target': (8, 17),
+ },
+}
\ No newline at end of file
diff --git a/src/openmc_data/utils.py b/src/openmc_data/utils.py
index 44eebd9..b458df2 100644
--- a/src/openmc_data/utils.py
+++ b/src/openmc_data/utils.py
@@ -215,6 +215,15 @@ def extract(
Path(compressed_file).unlink(missing_ok=True)
+def _file_md5(path):
+ """Hash large archives without loading them into memory."""
+ checksum = hashlib.md5()
+ with path.open('rb') as stream:
+ for chunk in iter(lambda: stream.read(_BLOCK_SIZE), b''):
+ checksum.update(chunk)
+ return checksum.hexdigest()
+
+
def download(
url: str,
checksum=None,
@@ -268,8 +277,9 @@ def download(
local_path = Path(output_path) / output_filename
# Check if file already downloaded
- if local_path.is_file():
- if local_path.stat().st_size == file_size:
+ if file_size is not None and local_path.is_file():
+ if (local_path.stat().st_size == file_size
+ and (checksum is None or _file_md5(local_path) == checksum)):
print(f'Skipping {local_path}, already downloaded')
return local_path
@@ -283,13 +293,19 @@ def download(
break
fh.write(chunk)
downloaded += len(chunk)
- status = '{:10} [{:3.2f}%]'.format(
- downloaded, downloaded * 100. / file_size)
+ if file_size:
+ status = '{:10} [{:3.2f}%]'.format(
+ downloaded, downloaded * 100. / file_size)
+ else:
+ status = f'{downloaded:10} bytes'
print(status + '\b'*len(status), end='', flush=True)
print('')
+ if file_size is not None and downloaded != file_size:
+ raise OSError(f"Incomplete download of {url}: expected {file_size} bytes, received {downloaded}")
+
if checksum is not None:
- downloadsum = hashlib.md5(open(local_path, 'rb').read()).hexdigest()
+ downloadsum = _file_md5(local_path)
if downloadsum != checksum:
raise OSError("MD5 checksum for {} does not match. If this is "
"your first time receiving this message, please "
diff --git a/tests/test_generate_jeff.py b/tests/test_generate_jeff.py
new file mode 100644
index 0000000..8d4ec5c
--- /dev/null
+++ b/tests/test_generate_jeff.py
@@ -0,0 +1,504 @@
+"""Offline orchestration tests and opt-in real NJOY sample tests.
+
+Offline tests use real HDF5/XML with mocked NJOY. Integration tests require
+OPENMC_JEFF_INTEGRATION=1 and NJOY on PATH, and stream bounded archive samples
+rather than saving complete libraries. They are not full-library qualification.
+"""
+
+import hashlib
+import importlib
+import os
+from pathlib import Path
+import struct
+import sys
+import tarfile
+import tempfile
+from types import SimpleNamespace
+from unittest.mock import MagicMock
+import zipfile
+import zlib
+
+import h5py
+import numpy as np
+import openmc.data
+import pytest
+import requests
+
+from openmc_data.generate import generate_jeff as gen
+from openmc_data.generate import generate_jeff33 as legacy
+from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements
+
+
+@pytest.fixture(autouse=True)
+def temporary_workdir(tmp_path, monkeypatch):
+ """Keep working files and temporary staging/NJOY files inside tmp_path."""
+ monkeypatch.chdir(tmp_path)
+ monkeypatch.setattr(tempfile, 'tempdir', str(tmp_path))
+ monkeypatch.setenv('TMPDIR', str(tmp_path))
+
+
+def put(path, content='input'):
+ path.parent.mkdir(parents=True, exist_ok=True)
+ path.write_text(content)
+ return path
+
+
+def hdf5(directory, name, particle):
+ with h5py.File(directory / f'{name}.h5', 'w') as f:
+ f.attrs['filetype'] = np.bytes_('data_' + particle)
+ f.create_group(name)
+
+
+def inputs(root, release):
+ pairs = jeff_endf_details[release]['thermal']['pairs']
+ for i, name in enumerate(sorted({n for n, _ in pairs})):
+ put(root / 'neutron' / 'nested' / name, f'N{i}')
+ for i, (_, name) in enumerate(pairs):
+ put(root / 'thermal' / 'nested' / name, f'c_material{i}')
+ for prefix in ['photoat', 'atom']:
+ for z, symbol in [(1, 'H'), (8, 'O')]:
+ put(root / 'photon' / prefix / f'{prefix}-{z:03}_{symbol}_000.endf', symbol)
+
+
+@pytest.fixture
+def processing(tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ calls = []
+ def neutron(path, output, libver, temperatures):
+ calls.append(('neutron', path, libver, temperatures))
+ hdf5(output, path.read_text(), 'neutron')
+ def thermal(path, tsl, output, libver):
+ calls.append(('thermal', path, tsl, libver))
+ hdf5(output, tsl.read_text(), 'thermal')
+ def photon(photo, atom):
+ assert photo.read_text() == atom.read_text()
+ calls.append(('photon', photo, atom))
+ name = photo.read_text()
+ def export(path, mode, libver):
+ assert mode == 'w'
+ hdf5(path.parent, name, 'photon')
+ return SimpleNamespace(name=name, export_to_hdf5=export)
+ monkeypatch.setattr(gen, 'process_neutron', neutron)
+ monkeypatch.setattr(gen, 'process_thermal', thermal)
+ monkeypatch.setattr(openmc.data.IncidentPhoton, 'from_endf', photon)
+ monkeypatch.setattr(gen, 'download', MagicMock(side_effect=AssertionError('Unexpected network access')))
+ return calls
+
+
+def offline(release, particles):
+ return ['-r', release, '-p', *particles, '--jobs', '1', '--no-download', '--no-extract']
+
+
+@pytest.mark.parametrize('release,count', [('3.3', 20), ('4.0', 128)])
+@pytest.mark.parametrize('particles', [('neutron',), ('thermal',), ('photon',), ('neutron', 'thermal', 'photon')])
+def test_release_particle_selection(tmp_path, processing, release, count, particles):
+ root = tmp_path / f'jeff-{release}-endf'
+ inputs(root, release)
+ destination = tmp_path / f'jeff-{release}-hdf5'
+ put(destination / 'neutron' / 'stale.h5', 'not HDF5')
+ gen.main(offline(release, particles) + ['--temperatures', '300', '700', '--libver', 'latest'])
+ library = openmc.data.DataLibrary.from_xml(destination / 'cross_sections.xml')
+ assert {entry['type'] for entry in library} == set(particles)
+ assert all(Path(entry['path']).is_file() for entry in library)
+ assert all('stale' not in entry['path'] and '.jeff-' not in entry['path'] for entry in library)
+ assert (destination / 'neutron' / 'stale.h5').read_text() == 'not HDF5'
+ assert len([c for c in processing if c[0] == 'thermal']) == (count if 'thermal' in particles else 0)
+ for call in processing:
+ if call[0] == 'neutron':
+ assert call[2:] == ('latest', [300.0, 700.0])
+ if call[0] == 'thermal':
+ assert len(call) == 4 # no neutron temperature override
+ assert not list(tmp_path.glob('.jeff-generate-*'))
+
+
+def test_defaults_and_legacy_import_safe(monkeypatch):
+ monkeypatch.setattr(sys, 'argv', ['program', '--not-a-generator-option'])
+ importlib.reload(gen)
+ importlib.reload(legacy)
+ args = gen._parser().parse_args([])
+ assert args.release == '4.0' and not args.use_tmpdir
+ assert args.temperatures == [250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0]
+ assert args.particles == ['neutron', 'thermal', 'photon']
+ args = gen._parser(legacy=True).parse_args([])
+ assert args.release == '3.3' and args.use_tmpdir
+ assert args.destination == Path('jeff33_hdf5')
+ main = MagicMock()
+ monkeypatch.setattr(legacy, '_main', main)
+ legacy.main(['--jobs', '2'])
+ main.assert_called_once_with(['--jobs', '2'], legacy=True)
+
+
+@pytest.mark.parametrize('argv', [
+ ['-r', '3.2'], ['--jobs', '0'], ['--jobs', '-1'], ['--temperatures', '0'],
+ ['--temperatures', '-5'], ['--temperatures', 'nan'], ['--temperatures', 'inf'],
+ ['--temperatures', '293.6', '294'], ['-p', 'electron'],
+ ['-d', 'cache/out', '--endf-dir', 'cache'],
+ ['--endf-dir', 'cache', '--download-dir', 'cache/download'],
+])
+def test_invalid_arguments(argv, tmp_path, monkeypatch):
+ monkeypatch.chdir(tmp_path)
+ with pytest.raises(SystemExit) as error:
+ gen.main(argv)
+ assert error.value.code == 2
+ assert not list(tmp_path.iterdir())
+
+
+@pytest.mark.parametrize('release', ['3.3', '4.0'])
+def test_thermal_download_dependency(tmp_path, processing, monkeypatch, release):
+ inputs(tmp_path / f'jeff-{release}-endf', release)
+ groups = []
+ def prepare(details, archives, directory, args):
+ groups.append(directory.name)
+ monkeypatch.setattr(gen, '_prepare', prepare)
+ gen.main(['-r', release, '-p', 'thermal', '--jobs', '1'])
+ assert groups == ['neutron', 'thermal']
+ assert {c[0] for c in processing} == {'thermal'}
+ assert not (tmp_path / f'jeff-{release}-hdf5/neutron').exists()
+
+
+@pytest.mark.parametrize('problem', ['empty', 'missing_neutron', 'missing_thermal', 'duplicate'])
+def test_missing_and_ambiguous_inputs(tmp_path, processing, problem):
+ root = tmp_path / 'jeff-4.0-endf'
+ if problem != 'empty':
+ inputs(root, '4.0')
+ n, t = jeff_endf_details['4.0']['thermal']['pairs'][0]
+ if problem == 'missing_neutron':
+ (root / 'neutron/nested' / n).unlink()
+ elif problem == 'missing_thermal':
+ (root / 'thermal/nested' / t).unlink()
+ else:
+ put(root / 'neutron/duplicate' / n)
+ with pytest.raises((FileNotFoundError, ValueError)):
+ gen.main(offline('4.0', ['thermal']))
+ assert not processing
+ assert not (tmp_path / 'jeff-4.0-hdf5/cross_sections.xml').exists()
+
+
+@pytest.mark.parametrize('problem', ['missing', 'mismatch', 'duplicate'])
+def test_photon_pair_validation(tmp_path, processing, problem):
+ root = tmp_path / 'jeff-4.0-endf'
+ inputs(root, '4.0')
+ atom = root / 'photon/atom/atom-001_H_000.endf'
+ if problem == 'missing':
+ atom.unlink()
+ elif problem == 'mismatch':
+ atom.rename(atom.with_name('atom-002_He_000.endf'))
+ else:
+ put(root / 'photon/duplicate' / atom.name, 'H')
+ with pytest.raises(ValueError):
+ gen.main(offline('4.0', ['photon']))
+ assert not processing
+
+
+@pytest.mark.parametrize('existing', [False, True])
+@pytest.mark.parametrize('failure', ['worker', 'collision', 'empty', 'invalid_hdf5'])
+def test_processing_failure_never_publishes(tmp_path, processing, monkeypatch, existing, failure):
+ root = tmp_path / 'jeff-4.0-endf'
+ inputs(root, '4.0')
+ dest = tmp_path / 'jeff-4.0-hdf5'
+ if existing:
+ put(dest / 'cross_sections.xml', 'old manifest')
+ put(dest / 'neutron/N0.h5', 'old data')
+ def broken(path, output, libver, temperatures):
+ if failure == 'worker':
+ raise ValueError('NJOY failed')
+ if failure == 'collision':
+ hdf5(output, 'same', 'neutron')
+ if failure == 'invalid_hdf5':
+ put(output / 'bad.h5', 'invalid')
+ monkeypatch.setattr(gen, 'process_neutron', broken)
+ with pytest.raises((RuntimeError, ValueError, OSError)):
+ gen.main(offline('4.0', ['neutron']) + ['--cleanup'])
+ if existing:
+ assert (dest / 'cross_sections.xml').read_text() == 'old manifest'
+ assert (dest / 'neutron/N0.h5').read_text() == 'old data'
+ else:
+ assert not dest.exists()
+ assert root.is_dir()
+ assert not list(tmp_path.glob('.jeff-generate-*'))
+
+
+def test_pool_exception_is_retrieved(tmp_path, processing, monkeypatch):
+ inputs(tmp_path / 'jeff-4.0-endf', '4.0')
+ pool = MagicMock()
+ pool.__enter__.return_value = pool
+ pool.apply_async.return_value.get.side_effect = RuntimeError('remote NJOY failure')
+ factory = MagicMock(return_value=pool)
+ monkeypatch.setattr(gen, 'Pool', factory)
+ with pytest.raises(RuntimeError, match='remote NJOY failure'):
+ gen.main(offline('4.0', ['neutron']) + ['--jobs', '2'])
+ factory.assert_called_once_with(processes=2)
+ pool.apply_async.return_value.get.assert_called_once()
+ pool.apply_async.return_value.wait.assert_not_called()
+ assert not (tmp_path / 'jeff-4.0-hdf5/cross_sections.xml').exists()
+
+
+@pytest.mark.parametrize('failure', [False, True])
+def test_cleanup_only_after_success(tmp_path, processing, monkeypatch, failure):
+ root = tmp_path / 'jeff-4.0-endf'
+ inputs(root, '4.0')
+ archive = put(tmp_path / 'jeff-4.0-download/photon/data.zip')
+ other = put(tmp_path / 'jeff-4.0-download/neutron/keep.zip')
+ monkeypatch.setattr(gen, '_prepare', lambda *args: None)
+ if failure:
+ monkeypatch.setattr(gen, '_generate', MagicMock(side_effect=RuntimeError('failed')))
+ with pytest.raises(RuntimeError):
+ gen.main(['-p', 'photon', '--jobs', '1', '--cleanup'])
+ else:
+ gen.main(['-p', 'photon', '--jobs', '1', '--cleanup'])
+ assert archive.exists() is failure
+ assert (root / 'photon').exists() is failure
+ assert other.exists() and (root / 'neutron').exists()
+
+
+def test_legacy_offline_tmpdir_uses_persistent_inputs(tmp_path, processing):
+ inputs(tmp_path / 'jeff-3.3-endf', '3.3')
+ cwd = Path.cwd()
+ with pytest.warns(UserWarning, match='ignored'):
+ legacy.main(['-p', 'photon', '--no-download', '--no-extract', '--jobs', '1', '--cleanup'])
+ assert Path.cwd() == cwd
+ assert (tmp_path / 'jeff33_hdf5/cross_sections.xml').is_file()
+ assert (tmp_path / 'jeff-3.3-endf/photon').exists()
+
+
+@pytest.mark.parametrize('failure', [False, True])
+def test_temporary_input_cleanup_and_cwd(tmp_path, processing, monkeypatch, failure):
+ roots = []
+ def prepare(details, archives, directory, args):
+ roots.append(directory.parent)
+ inputs(directory.parent, '4.0')
+ monkeypatch.setattr(gen, '_prepare', prepare)
+ if failure:
+ monkeypatch.setattr(gen, '_generate', MagicMock(side_effect=RuntimeError('failed')))
+ with pytest.raises(RuntimeError):
+ gen.main(['-p', 'photon', '--use-tmpdir', '--jobs', '1'])
+ else:
+ gen.main(['-p', 'photon', '--use-tmpdir', '--jobs', '1'])
+ assert roots and all(not root.exists() for root in roots)
+ assert Path.cwd() == tmp_path
+ assert not (tmp_path / 'jeff-4.0-download').exists()
+
+
+@pytest.mark.parametrize('download_enabled', [False, True])
+def test_archive_cache_checksum_and_fresh_extraction(tmp_path, monkeypatch, download_enabled):
+ archives = tmp_path / 'archives'
+ archives.mkdir()
+ archive = archives / 'library.zip'
+ with zipfile.ZipFile(archive, 'w') as z:
+ z.writestr('nested/H1.jeff', 'H1')
+ checksum = hashlib.md5(archive.read_bytes()).hexdigest()
+ details = {'base_url': 'https://example.invalid/', 'compressed_files': ['library.zip'],
+ 'checksums': [checksum], 'url_suffix': '/content'}
+ args = SimpleNamespace(download=download_enabled, extract=True)
+ network = MagicMock(side_effect=AssertionError('Valid cache must not access network'))
+ monkeypatch.setattr(gen, 'download', network)
+ target = tmp_path / 'inputs'
+ put(target / 'stale.jeff')
+ gen._prepare(details, archives, target, args)
+ assert (target / 'nested/H1.jeff').read_text() == 'H1'
+ assert not (target / 'stale.jeff').exists()
+ network.assert_not_called()
+ archive.write_bytes(b'corrupt')
+ if not download_enabled:
+ with pytest.raises(OSError, match='MD5'):
+ gen._prepare(details, archives, target, args)
+ else:
+ with pytest.raises(AssertionError, match='network'):
+ gen._prepare(details, archives, target, args)
+ assert network.call_args.kwargs['output_filename'] == 'library.zip'
+ assert network.call_args.args[0].endswith('library.zip/content')
+ assert (target / 'nested/H1.jeff').is_file()
+
+
+def test_missing_archive_is_actionable(tmp_path):
+ with pytest.raises(FileNotFoundError, match='enable --download'):
+ gen._prepare(jeff_endf_details['4.0']['neutron'], tmp_path / 'archives',
+ tmp_path / 'inputs', SimpleNamespace(download=False, extract=True))
+
+
+def test_replacements_do_not_rename_originals(tmp_path, processing, monkeypatch):
+ inputs(tmp_path / 'jeff-3.3-endf', '3.3')
+ for name, details in jeff33_replacements.items():
+ put(tmp_path / 'jeff-3.3-endf/neutron/nested' / name, 'original')
+ put(tmp_path / 'jeff-3.3-endf/replacements' / Path(name).stem / 'evaluation.dat', name[:1])
+ targets = {'6': (6, 13), '8': (8, 17)}
+ def evaluation(path):
+ z, a = targets[path.read_text()]
+ return SimpleNamespace(target={'atomic_number': z, 'mass_number': a, 'isomeric_state': 0})
+ monkeypatch.setattr(openmc.data.endf, 'Evaluation', evaluation)
+ gen.main(offline('3.3', ['neutron']))
+ assert len([c for c in processing if c[1].suffix == '.dat']) == 2
+ for name in jeff33_replacements:
+ assert (tmp_path / 'jeff-3.3-endf/neutron/nested' / name).read_text() == 'original'
+ assert (tmp_path / 'jeff-3.3-endf/replacements' / Path(name).stem / 'evaluation.dat').exists()
+
+
+@pytest.mark.parametrize('problem', ['missing', 'multiple', 'wrong_target', 'metastable'])
+def test_replacement_validation(tmp_path, monkeypatch, problem):
+ details = jeff33_replacements['6-C-13g.jeff33']
+ if problem != 'missing':
+ put(tmp_path / 'evaluation.dat')
+ if problem == 'multiple':
+ put(tmp_path / 'other.dat')
+ target = {'atomic_number': 8 if problem == 'wrong_target' else 6,
+ 'mass_number': 13, 'isomeric_state': int(problem == 'metastable')}
+ monkeypatch.setattr(openmc.data.endf, 'Evaluation', lambda _: SimpleNamespace(target=target))
+ with pytest.raises(ValueError):
+ gen._replacement(details, tmp_path)
+
+
+def test_publication_failure_removes_manifest(tmp_path, processing, monkeypatch):
+ inputs(tmp_path / 'jeff-4.0-endf', '4.0')
+ dest = tmp_path / 'jeff-4.0-hdf5'
+ put(dest / 'cross_sections.xml', 'old manifest')
+ replace = Path.replace
+ def fail(path, target):
+ if Path(target).parent == dest / 'photon':
+ raise OSError('disk failure')
+ return replace(path, target)
+ monkeypatch.setattr(Path, 'replace', fail)
+ with pytest.raises(OSError, match='disk failure'):
+ gen.main(offline('4.0', ['photon']))
+ assert not (dest / 'cross_sections.xml').exists()
+
+
+def test_metadata_invariants():
+ assert set(jeff_endf_details) == {'3.3', '4.0'}
+ for release, count in [('3.3', 20), ('4.0', 128)]:
+ details = jeff_endf_details[release]
+ assert set(details) == {'neutron', 'thermal', 'photon'}
+ pairs = details['thermal']['pairs']
+ assert len(pairs) == len({t for n, t in pairs}) == count
+ assert all(n.endswith(details['neutron']['pattern'][1:]) for n, _ in pairs)
+ for group in details.values():
+ assert len(group['compressed_files']) == len(group['checksums'])
+ assert all(len(checksum) == 32 for checksum in group['checksums'])
+ assert ('VIII.0' if release == '3.3' else 'VIII.1') in details['photon']['compressed_files'][0]
+ assert ('n_8-O-016g.jeff', 'tsl_O_D20.jeff') in jeff_endf_details['4.0']['thermal']['pairs']
+ assert len(jeff33_replacements) == 2
+
+
+# Opt-in real NJOY samples, separate from the offline orchestration above.
+class BoundedStream:
+ def __init__(self, raw, limit):
+ self.raw, self.limit, self.count = raw, limit, 0
+
+ def read(self, size):
+ if size < 0 or self.count + size > self.limit:
+ raise RuntimeError('Archive sample exceeded its byte budget')
+ data = self.raw.read(size)
+ self.count += len(data)
+ return data
+
+
+def sample(details, wanted, destination, budget):
+ """Extract named samples while streaming; close immediately after finding them."""
+ name = details['compressed_files'][0]
+ url = details['base_url'] + name + details.get('url_suffix', '')
+ missing = set(wanted)
+ destination.mkdir(parents=True, exist_ok=True)
+ with requests.get(url, stream=True, headers={'Accept-Encoding': 'identity'}, timeout=(10, 60)) as r:
+ r.raise_for_status()
+ stream = BoundedStream(r.raw, budget)
+ if name.endswith('.zip'):
+ while missing:
+ header = stream.read(30)
+ assert header[:4] == b'PK\x03\x04', 'Missing sample or unsupported ZIP header'
+ fields = struct.unpack('<4s5H3I2H', header)
+ flags, method, crc, compressed, uncompressed = fields[2], fields[3], *fields[6:9]
+ member = stream.read(fields[9]).decode()
+ extra = stream.read(fields[10])
+ assert not flags & 8, 'Streaming sample requires known member sizes'
+ if compressed == 0xffffffff:
+ tag, length = struct.unpack('= 16
+ uncompressed, compressed = struct.unpack(' 1 and data.temperatures != ['700K']
+ assert len(openmc.data.DataLibrary.from_xml(tmp_path / 'output/cross_sections.xml')) == 1
+
+
+@pytest.mark.network
+@pytest.mark.integration
+@pytest.mark.skipif(os.environ.get('OPENMC_JEFF_INTEGRATION') != '1',
+ reason='set OPENMC_JEFF_INTEGRATION=1 for opt-in network/NJOY sample validation')
+@pytest.mark.parametrize('release', ['3.3', '4.0'])
+def test_real_photon_pair(tmp_path, monkeypatch, release):
+ monkeypatch.chdir(tmp_path)
+ details = jeff_endf_details[release]['photon']
+ inputs = tmp_path / f'jeff-{release}-endf/photon'
+ for archive, prefix in zip(details['compressed_files'], ['photoat', 'atom']):
+ sample(dict(details, compressed_files=[archive]), [f'{prefix}-001_H_000.endf'],
+ inputs, 32 * 1024**2)
+ gen.main(['-r', release, '-p', 'photon', '--no-download', '--no-extract', '--jobs', '1'])
+ destination = tmp_path / f'jeff-{release}-hdf5'
+ data = openmc.data.IncidentPhoton.from_hdf5(destination / 'photon/H.h5')
+ assert data.atomic_number == 1
+ assert len(openmc.data.DataLibrary.from_xml(destination / 'cross_sections.xml')) == 1
\ No newline at end of file
diff --git a/tests/test_urls.py b/tests/test_urls.py
index 25002de..2801257 100644
--- a/tests/test_urls.py
+++ b/tests/test_urls.py
@@ -1,4 +1,6 @@
+import json
+import re
import time
from pathlib import Path
from unittest.mock import MagicMock
@@ -13,6 +15,7 @@
all_chain_release_details,
all_decay_release_details,
)
+from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements
# urls that serve a file kept in this repository, see check_url
@@ -270,3 +273,58 @@ def test_transient_failure_is_retried(http_response):
get.side_effect = [requests.Timeout('timed out'), response]
check_url('https://example.org/data')
assert get.call_count == 2
+
+
+def source_urls():
+ groups = [(f'{release}-{particle}', details)
+ for release, particles in jeff_endf_details.items()
+ for particle, details in particles.items()]
+ groups.extend((f'3.3-replacement-{name}', details) for name, details in jeff33_replacements.items())
+ for label, details in groups:
+ for name in details['compressed_files']:
+ yield pytest.param(details['base_url'] + name + details.get('url_suffix', ''),
+ id=f'{label}-{name}')
+
+
+@pytest.mark.parametrize('url', list(source_urls()))
+def test_generate_jeff_url_registry(url):
+ validate_url(url)
+
+
+@pytest.mark.network
+@pytest.mark.parametrize('url', list(source_urls()))
+def test_generate_jeff_urls(url):
+ check_url(url)
+
+
+def bounded_text(url):
+ with requests.get(url, stream=True, headers={'Accept-Encoding': 'identity'}, timeout=(10, 60)) as r:
+ r.raise_for_status()
+ content = r.raw.read(2_000_001)
+ assert len(content) <= 2_000_000, 'Metadata/preview unexpectedly large'
+ return content.decode()
+
+
+@pytest.mark.network
+@pytest.mark.parametrize('release', ['3.3', '4.0'])
+@pytest.mark.parametrize('particle', ['neutron', 'thermal'])
+def test_nea_record_checksums(release, particle):
+ details = jeff_endf_details[release][particle]
+ record = json.loads(bounded_text(details['base_url'].rsplit('/files/', 1)[0]))
+ for name, checksum in zip(details['compressed_files'], details['checksums']):
+ entry = record['files']['entries'][name]
+ assert entry['checksum'] == 'md5:' + checksum
+ assert entry['links']['content'] == details['base_url'] + name + '/content'
+
+
+@pytest.mark.network
+@pytest.mark.parametrize('particle,column', [('neutron', 0), ('thermal', 1)])
+def test_jeff40_pairs_exist_in_archive_preview(particle, column):
+ details = jeff_endf_details['4.0'][particle]
+ record = details['base_url'].split('/records/')[1].split('/')[0]
+ name = details['compressed_files'][0]
+ text = bounded_text(f'https://data.oecd-nea.org/records/{record}/preview/{name}')
+ members = re.findall(r'\s*([^<>]+\.jeff)', text)
+ assert len(members) >= (593 if particle == 'neutron' else 128)
+ required = {pair[column] for pair in jeff_endf_details['4.0']['thermal']['pairs']}
+ assert not required - set(members)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index ddd8482..6d9f928 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,3 +1,5 @@
+import hashlib
+import io
import tarfile
import pytest
@@ -29,3 +31,115 @@ def test_process_neutron_requires_openmc():
with pytest.raises(ModuleNotFoundError, match="openmc is required"):
openmc_data.process_neutron("dummy.endf", "out", "latest")
+
+
+class Response(io.BytesIO):
+ def __init__(self, payload, length):
+ super().__init__(payload)
+ self.length = length
+ self.read_sizes = []
+
+ def read(self, size=-1):
+ assert size == utils._BLOCK_SIZE, 'Download must use bounded reads'
+ self.read_sizes.append(size)
+ return super().read(size)
+
+
+@pytest.mark.parametrize('known_length', [False, True])
+@pytest.mark.parametrize('existing', [False, True])
+def test_download_streaming_length_and_cache(tmp_path, monkeypatch, capsys, known_length, existing):
+ payload = b'x' * (2*utils._BLOCK_SIZE + 17)
+ response = Response(payload, len(payload) if known_length else None)
+ target = tmp_path / 'library.zip'
+ if existing:
+ target.write_bytes(b'o' * len(payload))
+ calls = []
+ def urlopen(request, **kwargs):
+ calls.append((request, kwargs))
+ return response
+ monkeypatch.setattr(utils, 'urlopen', urlopen)
+ path = utils.download('https://example.invalid/library.zip/content', as_browser=True,
+ output_path=tmp_path, output_filename='library.zip', timeout=10)
+ assert path == target
+ assert calls[0][0].get_header('User-agent') == 'Mozilla/5.0'
+ assert calls[0][1] == {'timeout': 10}
+ if existing and known_length:
+ assert target.read_bytes() == b'o' * len(payload)
+ assert not response.read_sizes
+ else:
+ assert target.read_bytes() == payload
+ assert len(response.read_sizes) == 4
+ output = capsys.readouterr().out
+ if not known_length:
+ assert 'bytes' in output and '%' not in output and 'Skipping' not in output
+ assert response.closed
+
+
+@pytest.mark.parametrize('length', [None, 0])
+def test_empty_download(tmp_path, monkeypatch, length):
+ monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: Response(b'', length))
+ path = utils.download('https://example.invalid/empty', output_path=tmp_path)
+ assert path.read_bytes() == b''
+
+
+def test_wrong_cached_size_redownloaded(tmp_path, monkeypatch):
+ (tmp_path / 'file').write_bytes(b'partial')
+ monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: Response(b'new', 3))
+ assert utils.download('https://example.invalid/file', output_path=tmp_path).read_bytes() == b'new'
+
+
+@pytest.mark.parametrize('valid', [False, True])
+def test_unknown_length_checksum(tmp_path, monkeypatch, valid):
+ monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: Response(b'payload', None))
+ checksum = hashlib.md5(b'payload' if valid else b'wrong').hexdigest()
+ if valid:
+ assert utils.download('https://example.invalid/file', checksum=checksum, output_path=tmp_path).is_file()
+ else:
+ with pytest.raises(OSError, match='MD5 checksum'):
+ utils.download('https://example.invalid/file', checksum=checksum, output_path=tmp_path)
+
+
+@pytest.mark.parametrize('valid_cache', [False, True])
+def test_known_length_cached_checksum(tmp_path, monkeypatch, valid_cache):
+ payload = b'payload'
+ target = tmp_path / 'file'
+ target.write_bytes(payload if valid_cache else b'invalid')
+ response = Response(payload, len(payload))
+ monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: response)
+ checksum = hashlib.md5(payload).hexdigest()
+ assert utils.download('https://example.invalid/file', checksum=checksum,
+ output_path=tmp_path).read_bytes() == payload
+ assert bool(response.read_sizes) is not valid_cache
+ assert response.closed
+
+
+def test_checksum_uses_bounded_reads(tmp_path, monkeypatch):
+ from pathlib import Path
+
+ payload = b'x' * (2*utils._BLOCK_SIZE + 17)
+ response = Response(payload, None)
+ monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: response)
+ opened = Path.open
+ reads = []
+
+ def bounded(path, mode='r', *args, **kwargs):
+ if mode == 'rb':
+ stream = Response(payload, len(payload))
+ reads.append(stream)
+ return stream
+ return opened(path, mode, *args, **kwargs)
+
+ monkeypatch.setattr(Path, 'open', bounded)
+ utils.download('https://example.invalid/file', checksum=hashlib.md5(payload).hexdigest(),
+ output_path=tmp_path)
+ assert len(reads) == 1
+ assert len(reads[0].read_sizes) == 4
+ assert reads[0].closed
+
+
+def test_truncated_download_fails(tmp_path, monkeypatch):
+ response = Response(b'partial', 100)
+ monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: response)
+ with pytest.raises(OSError, match='Incomplete download'):
+ utils.download('https://example.invalid/file', output_path=tmp_path)
+ assert response.closed
From f18cd08075adaa1836783787442518fd1fe189b0 Mon Sep 17 00:00:00 2001
From: shimwell <8583900+shimwell@users.noreply.github.com>
Date: Tue, 20 Oct 2026 14:01:01 +0200
Subject: [PATCH 2/5] Simplify JEFF staging and consolidate generation tests
---
README.md | 114 +++++---------
pyproject.toml | 3 +-
src/openmc_data/generate/generate_jeff.py | 22 +--
tests/test_generate_jeff.py | 174 ++++++++--------------
tests/test_utils.py | 102 +++++--------
5 files changed, 142 insertions(+), 273 deletions(-)
diff --git a/README.md b/README.md
index 3b70898..36b6f94 100644
--- a/README.md
+++ b/README.md
@@ -107,84 +107,42 @@ A few categories of scripts are available:
#### JEFF generation
-`generate_jeff` processes ENDF evaluations with OpenMC and NJOY, defaulting to
-JEFF 4.0. Use `--release 3.3` for JEFF 3.3, and `--particles neutron thermal
-photon` to select particle types (all three are selected by default).
-Thermal-only generation downloads the neutron evaluations it needs, but does
-not produce neutron HDF5 files. `--jobs` controls parallel workers and defaults
-to the CPU count; `--jobs 1` processes serially.
-
-For example, `generate_jeff --release 4.0 --particles neutron --temperatures
-293.6 600 --jobs 4` generates neutron data at two temperatures. The default
-neutron temperatures are 250, 293.6, 600, 900, 1200 and 2500 K. Thermal
-scattering always uses the evaluation's native temperatures, regardless of
-`--temperatures`. NJOY must be on `PATH`. `--libver earliest` (default) and
-`--libver latest` control HDF5 compatibility.
-
-Downloads come from the NEA records for
-[JEFF 3.3 neutrons](https://data.oecd-nea.org/records/bh7jn-rm903),
-[JEFF 3.3 thermal scattering](https://data.oecd-nea.org/records/3nm84-46r21),
-[JEFF 4.0 neutrons](https://data.oecd-nea.org/records/e9ajn-a3p20), and
-[JEFF 4.0 thermal scattering](https://data.oecd-nea.org/records/scmva-nqh68).
-Published MD5 checksums are checked, including when extracting a local cache.
-Photon data are supplements from ENDF/B-VIII.0 for JEFF 3.3 and ENDF/B-VIII.1
-for JEFF 4.0, not JEFF evaluations.
-
-JEFF 3.3 C13 and O17 neutron evaluations are replaced with the individual
-[TENDL-2019 archives published by IAEA](https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/).
-The original JEFF files remain untouched. Each replacement ZIP must contain
-one evaluation with the expected ENDF target. IAEA does not publish checksums
-for these ZIPs; extraction checks ZIP CRCs. Some networks receive a Cloudflare
-challenge from IAEA. This is a download failure, not a successful generation;
-in that case, obtain the archives manually and reuse a local cache.
-
-JEFF 3.3 retains the existing 20 thermal pairs. JEFF 4.0 uses the 128 thermal
-pairs in the [upstream generator](https://github.com/openmc-dev/data/blob/master/generate_jeff.py),
-checked against NEA's archive inventories. Additional temperature-split
-PuO2/UO2/Zy4 evaluations in the JEFF 4.0 archive are not included. Availability
-in an archive does not guarantee that every evaluation is processable by a
-particular OpenMC/NJOY version; full-library NJOY qualification is not part of
-the automated tests.
-
-The default destination is `jeff-RELEASE-hdf5`, with particle subdirectories
-and a relative-path XML library manifest. `--destination` overrides it. Each
-worker writes to isolated staging storage; missing inputs, duplicate names,
-unpaired photon inputs, or worker errors abort generation. Only outputs from
-the current run are registered, never stale files already in the destination.
-Processing or validation failures leave an existing library unchanged. During
-publication the old manifest is removed before replacing data, and the new
-manifest is installed last, so a publication failure cannot advertise mixed
-data as a successful library.
-
-Archive and extracted-input directories default to `jeff-RELEASE-download`
-and `jeff-RELEASE-endf`. Override these with `--download-dir` and `--endf-dir`.
-Both use `neutron`, `thermal`, and `photon` subdirectories. NEA archives retain
-their original basenames despite the download URLs ending in `/content`.
-JEFF 3.3 replacement archives and extracted files go into separate
-`replacements/6-C-13g` and `replacements/8-O-17g` subdirectories of those roots.
-The output and two cache roots must not overlap.
-
-- `--no-download`: extract cached archives without accessing the network.
-- `--no-download --no-extract`: process local inputs in the particle
- subdirectories, including nested archive folders. Neutron/photon subsets are
- allowed; thermal generation requires all mapped pairs for the release.
-- `--cleanup`: after success, remove only the selected particle/replacement
- cache directories for enabled download/extraction stages. Offline inputs are
- retained when both stages are disabled.
-- `--use-tmpdir`: use automatically cleaned temporary inputs for a fresh
- download and extraction. When reusing inputs or specifying cache roots, this
- option is ignored with a warning instead of losing access to local files.
-
-`generate_jeff33` remains an import-safe compatibility wrapper. It fixes the
-release to 3.3 and keeps its legacy destination `jeff33_hdf5` and temporary-input
-default. Reusable inputs use the new release-scoped directory layout described
-above, not the legacy working-directory layout.
-
-Offline orchestration tests use real HDF5/XML with mocked NJOY. Separate live
-source tests check archive availability, NEA checksums, and the 4.0 inventory.
-Opt-in NJOY smoke tests use bounded streamed samples rather than downloading
-complete libraries; set `OPENMC_JEFF_INTEGRATION=1` with NJOY on `PATH` when
-running the integration tests.
+`generate_jeff -r 4.0 -p neutron --temperatures 293.6 600 --jobs 4` processes
+ENDF with OpenMC and NJOY (`njoy` must be on `PATH`). Defaults: release 4.0,
+all three particles, CPU-count workers, `--libver earliest`, and neutron
+temperatures 250, 293.6, 600, 900, 1200 and 2500 K. Thermal temperatures are
+native; thermal-only runs fetch neutron inputs without generating neutron HDF5.
+
+NEA sources and published MD5 checksums are in [src/openmc_data/urls_jeff.py](src/openmc_data/urls_jeff.py).
+Photon supplements use ENDF/B-VIII.0 for JEFF 3.3 and VIII.1 for 4.0.
+The 20 legacy 3.3 and 128 [upstream 4.0 thermal pairs](https://github.com/openmc-dev/data/blob/master/generate_jeff.py)
+are supported; additional temperature-split PuO2/UO2/Zy4 evaluations are excluded.
+Full-library NJOY qualification has not been performed.
+
+For 3.3, C13/O17 use individual [IAEA TENDL-2019 replacements](https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/)
+without editing originals. ZIP CRCs and ENDF targets are checked (no published MD5).
+If IAEA blocks automated access, manually cache the archives; blocked downloads fail.
+
+Output goes into `--destination` (default `jeff-RELEASE-hdf5`) with particle
+subdirectories and relative XML paths. Isolated workers stage and validate outputs
+before publication; only current outputs are registered. Processing failures preserve
+existing data; publication removes the old manifest first and installs the new one last.
+
+Caches default to `jeff-RELEASE-download` and `jeff-RELEASE-endf`, overridable via
+`--download-dir`/`--endf-dir`. All three roots must be disjoint. Particle subdirectories
+hold original archive basenames and nested inputs; 3.3 replacements use
+`replacements/6-C-13g` and `replacements/8-O-17g` beneath each cache root.
+- `--no-download` extracts cached archives, checking published MD5s.
+- `--no-download --no-extract` processes local inputs. Neutron/photon subsets are
+ allowed; thermal runs require every mapped pair.
+- `--cleanup` removes selected caches only after success and only for enabled stages.
+- `--use-tmpdir` uses temporary inputs for fresh downloads/extraction; it is ignored
+ with a warning for reused inputs or explicit cache roots.
+
+`generate_jeff33` keeps release 3.3, destination `jeff33_hdf5`, and temporary-input
+defaults, but reusable inputs follow the new layout. See `generate_jeff --help`.
+Tests use real HDF5/XML with mocked NJOY; opt-in bounded real-data samples require
+`OPENMC_JEFF_INTEGRATION=1` and NJOY. Live URL checks run separately.
### Download cross sections
diff --git a/pyproject.toml b/pyproject.toml
index e1bc05b..4ea8920 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,7 +34,8 @@ write_to = "src/openmc_data/_version.py"
[project.optional-dependencies]
tests = [
"pytest",
- "requests"
+ "requests",
+ "stream-unzip",
]
[tool.pytest.ini_options]
diff --git a/src/openmc_data/generate/generate_jeff.py b/src/openmc_data/generate/generate_jeff.py
index 95ca00a..9d8c514 100644
--- a/src/openmc_data/generate/generate_jeff.py
+++ b/src/openmc_data/generate/generate_jeff.py
@@ -41,9 +41,8 @@ def _parser(legacy=False):
('cleanup', False, 'Remove selected caches after success, only for enabled download/extract stages'),
('use-tmpdir', legacy, 'Use temporary inputs for a fresh download and extraction'),
]:
- parser.add_argument('--' + flag, dest=flag.replace('-', '_'), action='store_true', help=help_text)
+ parser.add_argument('--' + flag, action='store_true', default=default, help=help_text)
parser.add_argument('--no-' + flag, dest=flag.replace('-', '_'), action='store_false')
- parser.set_defaults(**{flag.replace('-', '_'): default})
return parser
@@ -155,19 +154,14 @@ def _generate(tasks, destination, args):
results = [result.get() for result in pending]
library = openmc.data.DataLibrary()
- outputs = []
- seen = set()
+ outputs = {}
for particle, path in sorted(results, key=lambda item: (item[0], item[1].name)):
relative = Path(particle) / path.name
- if relative in seen:
+ if relative in outputs:
raise ValueError(f'Output name collision: {relative}')
- seen.add(relative)
- target = staging / 'library' / relative
- target.parent.mkdir(parents=True, exist_ok=True)
- path.replace(target)
- library.register_file(target)
- outputs.append(relative)
- manifest = staging / 'library' / 'cross_sections.xml'
+ library.register_file(path)
+ outputs[relative] = path
+ manifest = staging / 'cross_sections.xml'
library.export_to_xml(manifest)
# Make paths relative to the manifest, not the temporary staging directory.
@@ -186,10 +180,10 @@ def _generate(tasks, destination, args):
# Old data remain untouched on processing/validation failure. Once publishing
# starts, remove the old manifest so an I/O failure cannot advertise mixed data.
(destination / 'cross_sections.xml').unlink(missing_ok=True)
- for relative in outputs:
+ for relative, path in outputs.items():
target = destination / relative
target.parent.mkdir(parents=True, exist_ok=True)
- (staging / 'library' / relative).replace(target)
+ path.replace(target)
manifest.replace(destination / 'cross_sections.xml')
diff --git a/tests/test_generate_jeff.py b/tests/test_generate_jeff.py
index 8d4ec5c..3829900 100644
--- a/tests/test_generate_jeff.py
+++ b/tests/test_generate_jeff.py
@@ -5,24 +5,24 @@
rather than saving complete libraries. They are not full-library qualification.
"""
+from contextlib import nullcontext
import hashlib
import importlib
import os
from pathlib import Path
-import struct
import sys
import tarfile
import tempfile
from types import SimpleNamespace
from unittest.mock import MagicMock
import zipfile
-import zlib
import h5py
import numpy as np
import openmc.data
import pytest
import requests
+from stream_unzip import stream_unzip
from openmc_data.generate import generate_jeff as gen
from openmc_data.generate import generate_jeff33 as legacy
@@ -62,7 +62,6 @@ def inputs(root, release):
@pytest.fixture
def processing(tmp_path, monkeypatch):
- monkeypatch.chdir(tmp_path)
calls = []
def neutron(path, output, libver, temperatures):
calls.append(('neutron', path, libver, temperatures))
@@ -136,7 +135,6 @@ def test_defaults_and_legacy_import_safe(monkeypatch):
['--endf-dir', 'cache', '--download-dir', 'cache/download'],
])
def test_invalid_arguments(argv, tmp_path, monkeypatch):
- monkeypatch.chdir(tmp_path)
with pytest.raises(SystemExit) as error:
gen.main(argv)
assert error.value.code == 2
@@ -146,12 +144,10 @@ def test_invalid_arguments(argv, tmp_path, monkeypatch):
@pytest.mark.parametrize('release', ['3.3', '4.0'])
def test_thermal_download_dependency(tmp_path, processing, monkeypatch, release):
inputs(tmp_path / f'jeff-{release}-endf', release)
- groups = []
- def prepare(details, archives, directory, args):
- groups.append(directory.name)
+ prepare = MagicMock()
monkeypatch.setattr(gen, '_prepare', prepare)
gen.main(['-r', release, '-p', 'thermal', '--jobs', '1'])
- assert groups == ['neutron', 'thermal']
+ assert [call.args[2].name for call in prepare.call_args_list] == ['neutron', 'thermal']
assert {c[0] for c in processing} == {'thermal'}
assert not (tmp_path / f'jeff-{release}-hdf5/neutron').exists()
@@ -233,22 +229,28 @@ def test_pool_exception_is_retrieved(tmp_path, processing, monkeypatch):
assert not (tmp_path / 'jeff-4.0-hdf5/cross_sections.xml').exists()
+@pytest.mark.parametrize('temporary', [False, True])
@pytest.mark.parametrize('failure', [False, True])
-def test_cleanup_only_after_success(tmp_path, processing, monkeypatch, failure):
+def test_input_cleanup(tmp_path, processing, monkeypatch, temporary, failure):
root = tmp_path / 'jeff-4.0-endf'
inputs(root, '4.0')
archive = put(tmp_path / 'jeff-4.0-download/photon/data.zip')
other = put(tmp_path / 'jeff-4.0-download/neutron/keep.zip')
- monkeypatch.setattr(gen, '_prepare', lambda *args: None)
+ roots = []
+ def prepare(details, archives, directory, args):
+ roots.append(directory.parent)
+ inputs(directory.parent, '4.0')
+ monkeypatch.setattr(gen, '_prepare', prepare)
if failure:
monkeypatch.setattr(gen, '_generate', MagicMock(side_effect=RuntimeError('failed')))
- with pytest.raises(RuntimeError):
- gen.main(['-p', 'photon', '--jobs', '1', '--cleanup'])
- else:
- gen.main(['-p', 'photon', '--jobs', '1', '--cleanup'])
- assert archive.exists() is failure
- assert (root / 'photon').exists() is failure
+ with pytest.raises(RuntimeError) if failure else nullcontext():
+ gen.main(['-p', 'photon', '--jobs', '1', '--use-tmpdir' if temporary else '--cleanup'])
+ assert roots
+ assert all(path.exists() is not temporary for path in roots)
+ assert archive.exists() is (temporary or failure)
+ assert (root / 'photon').exists() is (temporary or failure)
assert other.exists() and (root / 'neutron').exists()
+ assert Path.cwd() == tmp_path
def test_legacy_offline_tmpdir_uses_persistent_inputs(tmp_path, processing):
@@ -261,24 +263,6 @@ def test_legacy_offline_tmpdir_uses_persistent_inputs(tmp_path, processing):
assert (tmp_path / 'jeff-3.3-endf/photon').exists()
-@pytest.mark.parametrize('failure', [False, True])
-def test_temporary_input_cleanup_and_cwd(tmp_path, processing, monkeypatch, failure):
- roots = []
- def prepare(details, archives, directory, args):
- roots.append(directory.parent)
- inputs(directory.parent, '4.0')
- monkeypatch.setattr(gen, '_prepare', prepare)
- if failure:
- monkeypatch.setattr(gen, '_generate', MagicMock(side_effect=RuntimeError('failed')))
- with pytest.raises(RuntimeError):
- gen.main(['-p', 'photon', '--use-tmpdir', '--jobs', '1'])
- else:
- gen.main(['-p', 'photon', '--use-tmpdir', '--jobs', '1'])
- assert roots and all(not root.exists() for root in roots)
- assert Path.cwd() == tmp_path
- assert not (tmp_path / 'jeff-4.0-download').exists()
-
-
@pytest.mark.parametrize('download_enabled', [False, True])
def test_archive_cache_checksum_and_fresh_extraction(tmp_path, monkeypatch, download_enabled):
archives = tmp_path / 'archives'
@@ -401,35 +385,19 @@ def sample(details, wanted, destination, budget):
r.raise_for_status()
stream = BoundedStream(r.raw, budget)
if name.endswith('.zip'):
- while missing:
- header = stream.read(30)
- assert header[:4] == b'PK\x03\x04', 'Missing sample or unsupported ZIP header'
- fields = struct.unpack('<4s5H3I2H', header)
- flags, method, crc, compressed, uncompressed = fields[2], fields[3], *fields[6:9]
- member = stream.read(fields[9]).decode()
- extra = stream.read(fields[10])
- assert not flags & 8, 'Streaming sample requires known member sizes'
- if compressed == 0xffffffff:
- tag, length = struct.unpack('= 16
- uncompressed, compressed = struct.unpack(' 1 and data.temperatures != ['700K']
- assert len(openmc.data.DataLibrary.from_xml(tmp_path / 'output/cross_sections.xml')) == 1
-
-
-@pytest.mark.network
-@pytest.mark.integration
-@pytest.mark.skipif(os.environ.get('OPENMC_JEFF_INTEGRATION') != '1',
- reason='set OPENMC_JEFF_INTEGRATION=1 for opt-in network/NJOY sample validation')
-@pytest.mark.parametrize('release', ['3.3', '4.0'])
-def test_real_photon_pair(tmp_path, monkeypatch, release):
- monkeypatch.chdir(tmp_path)
- details = jeff_endf_details[release]['photon']
- inputs = tmp_path / f'jeff-{release}-endf/photon'
- for archive, prefix in zip(details['compressed_files'], ['photoat', 'atom']):
- sample(dict(details, compressed_files=[archive]), [f'{prefix}-001_H_000.endf'],
- inputs, 32 * 1024**2)
- gen.main(['-r', release, '-p', 'photon', '--no-download', '--no-extract', '--jobs', '1'])
+def test_real_sample(tmp_path, release, particle, n, t):
+ root = tmp_path / f'jeff-{release}-endf'
destination = tmp_path / f'jeff-{release}-hdf5'
- data = openmc.data.IncidentPhoton.from_hdf5(destination / 'photon/H.h5')
- assert data.atomic_number == 1
+ details = jeff_endf_details[release]
+ if n:
+ sample(details['neutron'], [n], root / 'neutron', 128 * 1024**2)
+ if t:
+ sample(details['thermal'], [t], root / 'thermal', 40 * 1024**2)
+ args = gen._parser().parse_args(['--jobs', '1', '--temperatures', '700'])
+ gen._generate([('thermal', (root / 'neutron' / n, root / 'thermal' / t))], destination, args)
+ else:
+ if particle == 'photon':
+ for archive, prefix in zip(details['photon']['compressed_files'], ['photoat', 'atom']):
+ sample(dict(details['photon'], compressed_files=[archive]),
+ [f'{prefix}-001_H_000.endf'], root / 'photon', 32 * 1024**2)
+ # Real neutron workers exercise child-process import safety and error transport.
+ gen.main(offline(release, [particle]) + ['--temperatures', '293.6',
+ '--jobs', '2' if n else '1'])
+ outputs = list((destination / particle).glob('*.h5'))
+ assert len(outputs) == 1
+ if particle == 'neutron':
+ data = openmc.data.IncidentNeutron.from_hdf5(outputs[0])
+ assert data.name == 'H1' and '294K' in data.temperatures
+ elif particle == 'thermal':
+ data = openmc.data.ThermalScattering.from_hdf5(outputs[0])
+ assert len(data.temperatures) > 1 and data.temperatures != ['700K']
+ else:
+ assert openmc.data.IncidentPhoton.from_hdf5(outputs[0]).atomic_number == 1
assert len(openmc.data.DataLibrary.from_xml(destination / 'cross_sections.xml')) == 1
\ No newline at end of file
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 6d9f928..4ef8b4f 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,6 +1,8 @@
import hashlib
import io
+from pathlib import Path
import tarfile
+from unittest.mock import Mock
import pytest
@@ -53,16 +55,13 @@ def test_download_streaming_length_and_cache(tmp_path, monkeypatch, capsys, know
target = tmp_path / 'library.zip'
if existing:
target.write_bytes(b'o' * len(payload))
- calls = []
- def urlopen(request, **kwargs):
- calls.append((request, kwargs))
- return response
+ urlopen = Mock(return_value=response)
monkeypatch.setattr(utils, 'urlopen', urlopen)
path = utils.download('https://example.invalid/library.zip/content', as_browser=True,
output_path=tmp_path, output_filename='library.zip', timeout=10)
assert path == target
- assert calls[0][0].get_header('User-agent') == 'Mozilla/5.0'
- assert calls[0][1] == {'timeout': 10}
+ assert urlopen.call_args.args[0].get_header('User-agent') == 'Mozilla/5.0'
+ assert urlopen.call_args.kwargs == {'timeout': 10}
if existing and known_length:
assert target.read_bytes() == b'o' * len(payload)
assert not response.read_sizes
@@ -75,71 +74,38 @@ def urlopen(request, **kwargs):
assert response.closed
-@pytest.mark.parametrize('length', [None, 0])
-def test_empty_download(tmp_path, monkeypatch, length):
- monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: Response(b'', length))
- path = utils.download('https://example.invalid/empty', output_path=tmp_path)
- assert path.read_bytes() == b''
-
-
-def test_wrong_cached_size_redownloaded(tmp_path, monkeypatch):
- (tmp_path / 'file').write_bytes(b'partial')
- monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: Response(b'new', 3))
- assert utils.download('https://example.invalid/file', output_path=tmp_path).read_bytes() == b'new'
-
-
-@pytest.mark.parametrize('valid', [False, True])
-def test_unknown_length_checksum(tmp_path, monkeypatch, valid):
- monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: Response(b'payload', None))
- checksum = hashlib.md5(b'payload' if valid else b'wrong').hexdigest()
- if valid:
- assert utils.download('https://example.invalid/file', checksum=checksum, output_path=tmp_path).is_file()
- else:
- with pytest.raises(OSError, match='MD5 checksum'):
- utils.download('https://example.invalid/file', checksum=checksum, output_path=tmp_path)
-
-
-@pytest.mark.parametrize('valid_cache', [False, True])
-def test_known_length_cached_checksum(tmp_path, monkeypatch, valid_cache):
- payload = b'payload'
+@pytest.mark.parametrize('payload,length,cached,checksum_data,error', [
+ (b'', None, None, None, None), (b'', 0, None, None, None),
+ (b'new', 3, b'partial', None, None),
+ (b'payload', None, None, b'payload', None),
+ (b'payload', None, None, b'wrong', 'MD5 checksum'),
+ (b'payload', 7, b'payload', b'payload', None),
+ (b'payload', 7, b'invalid', b'payload', None),
+ (b'partial', 100, None, None, 'Incomplete download'),
+])
+def test_download_validation(tmp_path, monkeypatch, payload, length, cached, checksum_data, error):
target = tmp_path / 'file'
- target.write_bytes(payload if valid_cache else b'invalid')
- response = Response(payload, len(payload))
- monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: response)
- checksum = hashlib.md5(payload).hexdigest()
- assert utils.download('https://example.invalid/file', checksum=checksum,
- output_path=tmp_path).read_bytes() == payload
- assert bool(response.read_sizes) is not valid_cache
+ if cached is not None:
+ target.write_bytes(cached)
+ response = Response(payload, length)
+ monkeypatch.setattr(utils, 'urlopen', Mock(return_value=response))
+ checksum = hashlib.md5(checksum_data).hexdigest() if checksum_data is not None else None
+ if error:
+ with pytest.raises(OSError, match=error):
+ utils.download('https://example.invalid/file', checksum=checksum, output_path=tmp_path)
+ else:
+ assert utils.download('https://example.invalid/file', checksum=checksum,
+ output_path=tmp_path).read_bytes() == payload
+ assert bool(response.read_sizes) is not (cached == payload)
assert response.closed
def test_checksum_uses_bounded_reads(tmp_path, monkeypatch):
- from pathlib import Path
-
payload = b'x' * (2*utils._BLOCK_SIZE + 17)
- response = Response(payload, None)
- monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: response)
- opened = Path.open
- reads = []
-
- def bounded(path, mode='r', *args, **kwargs):
- if mode == 'rb':
- stream = Response(payload, len(payload))
- reads.append(stream)
- return stream
- return opened(path, mode, *args, **kwargs)
-
- monkeypatch.setattr(Path, 'open', bounded)
- utils.download('https://example.invalid/file', checksum=hashlib.md5(payload).hexdigest(),
- output_path=tmp_path)
- assert len(reads) == 1
- assert len(reads[0].read_sizes) == 4
- assert reads[0].closed
-
-
-def test_truncated_download_fails(tmp_path, monkeypatch):
- response = Response(b'partial', 100)
- monkeypatch.setattr(utils, 'urlopen', lambda *a, **k: response)
- with pytest.raises(OSError, match='Incomplete download'):
- utils.download('https://example.invalid/file', output_path=tmp_path)
- assert response.closed
+ stream = Response(payload, len(payload))
+ opened = Mock(return_value=stream)
+ monkeypatch.setattr(Path, 'open', opened)
+ assert utils._file_md5(tmp_path / 'file') == hashlib.md5(payload).hexdigest()
+ opened.assert_called_once_with('rb')
+ assert len(stream.read_sizes) == 4
+ assert stream.closed
From 43445baf7cec7022e9b71e5aecfc646e5667c103 Mon Sep 17 00:00:00 2001
From: shimwell <8583900+shimwell@users.noreply.github.com>
Date: Tue, 20 Oct 2026 16:31:02 +0200
Subject: [PATCH 3/5] Remove legacy JEFF generation entry point
---
.github/workflows/test_package.yml | 1 -
README.md | 4 +---
pyproject.toml | 1 -
src/openmc_data/generate/generate_jeff.py | 18 ++++++----------
src/openmc_data/generate/generate_jeff33.py | 12 -----------
tests/test_generate_jeff.py | 23 +++++++--------------
6 files changed, 15 insertions(+), 44 deletions(-)
delete mode 100755 src/openmc_data/generate/generate_jeff33.py
diff --git a/.github/workflows/test_package.yml b/.github/workflows/test_package.yml
index 27c9848..2cfab87 100644
--- a/.github/workflows/test_package.yml
+++ b/.github/workflows/test_package.yml
@@ -64,7 +64,6 @@ jobs:
generate_cendl --help
generate_endf --help
generate_jeff --help
- generate_jeff33 --help
generate_jendl --help
- name: Test download h5 scripts exist
diff --git a/README.md b/README.md
index 36b6f94..1b19821 100644
--- a/README.md
+++ b/README.md
@@ -102,7 +102,6 @@ A few categories of scripts are available:
| generate_fendl | FENDL | 3.2c
3.2b
3.2a
3.2
3.1d
3.1a
3.0 | |
| generate_jendl | JENDL | 4.0
5.0 | |
| generate_jeff | JEFF | 3.3
4.0 | NJOY |
-| generate_jeff33 | JEFF | 3.3 (legacy alias) | NJOY |
| generate_tendl | TENDL | 2023
2025 | |
#### JEFF generation
@@ -139,8 +138,7 @@ hold original archive basenames and nested inputs; 3.3 replacements use
- `--use-tmpdir` uses temporary inputs for fresh downloads/extraction; it is ignored
with a warning for reused inputs or explicit cache roots.
-`generate_jeff33` keeps release 3.3, destination `jeff33_hdf5`, and temporary-input
-defaults, but reusable inputs follow the new layout. See `generate_jeff --help`.
+See `generate_jeff --help` for all options.
Tests use real HDF5/XML with mocked NJOY; opt-in bounded real-data samples require
`OPENMC_JEFF_INTEGRATION=1` and NJOY. Live URL checks run separately.
diff --git a/pyproject.toml b/pyproject.toml
index 4ea8920..2952aa9 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -73,7 +73,6 @@ convert_tendl = "openmc_data.convert.convert_tendl:main"
generate_cendl = "openmc_data.generate.generate_cendl:main"
generate_endf = "openmc_data.generate.generate_endf:main"
generate_jeff = "openmc_data.generate.generate_jeff:main"
-generate_jeff33 = "openmc_data.generate.generate_jeff33:main"
generate_jendl = "openmc_data.generate.generate_jendl:main"
generate_fendl = "openmc_data.generate.generate_fendl:main"
generate_tendl = "openmc_data.generate.generate_tendl:main"
diff --git a/src/openmc_data/generate/generate_jeff.py b/src/openmc_data/generate/generate_jeff.py
index 9d8c514..2abb24a 100644
--- a/src/openmc_data/generate/generate_jeff.py
+++ b/src/openmc_data/generate/generate_jeff.py
@@ -19,13 +19,11 @@
from openmc_data.utils import _file_md5
-def _parser(legacy=False):
+def _parser():
parser = argparse.ArgumentParser(description=__doc__,
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument('-r', '--release', choices=['3.3'] if legacy else ['3.3', '4.0'],
- default='3.3' if legacy else '4.0')
- parser.add_argument('-d', '--destination', type=Path,
- default=Path('jeff33_hdf5') if legacy else None)
+ parser.add_argument('-r', '--release', choices=['3.3', '4.0'], default='4.0')
+ parser.add_argument('-d', '--destination', type=Path, help='Output directory (default: jeff-RELEASE-hdf5)')
parser.add_argument('-p', '--particles', nargs='+', choices=['neutron', 'thermal', 'photon'],
default=['neutron', 'thermal', 'photon'])
parser.add_argument('--temperatures', nargs='+', type=float,
@@ -39,7 +37,7 @@ def _parser(legacy=False):
('download', True, 'Download missing or invalid archives'),
('extract', True, 'Extract archives into particle subdirectories'),
('cleanup', False, 'Remove selected caches after success, only for enabled download/extract stages'),
- ('use-tmpdir', legacy, 'Use temporary inputs for a fresh download and extraction'),
+ ('use-tmpdir', False, 'Use temporary inputs for a fresh download and extraction'),
]:
parser.add_argument('--' + flag, action='store_true', default=default, help=help_text)
parser.add_argument('--no-' + flag, dest=flag.replace('-', '_'), action='store_false')
@@ -187,8 +185,8 @@ def _generate(tasks, destination, args):
manifest.replace(destination / 'cross_sections.xml')
-def _main(argv=None, *, legacy=False):
- parser = _parser(legacy)
+def main(argv=None):
+ parser = _parser()
args = parser.parse_args(argv)
if args.jobs is not None and args.jobs < 1:
parser.error('--jobs must be positive')
@@ -250,9 +248,5 @@ def _main(argv=None, *, legacy=False):
print(f'Wrote {destination / "cross_sections.xml"}')
-def main(argv=None):
- _main(argv)
-
-
if __name__ == '__main__':
main()
\ No newline at end of file
diff --git a/src/openmc_data/generate/generate_jeff33.py b/src/openmc_data/generate/generate_jeff33.py
deleted file mode 100755
index f380d47..0000000
--- a/src/openmc_data/generate/generate_jeff33.py
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env python3
-"""Legacy JEFF 3.3 entry point; implementation lives in generate_jeff."""
-
-from .generate_jeff import _main
-
-
-def main(argv=None):
- _main(argv, legacy=True)
-
-
-if __name__ == '__main__':
- main()
diff --git a/tests/test_generate_jeff.py b/tests/test_generate_jeff.py
index 3829900..bda476c 100644
--- a/tests/test_generate_jeff.py
+++ b/tests/test_generate_jeff.py
@@ -25,7 +25,6 @@
from stream_unzip import stream_unzip
from openmc_data.generate import generate_jeff as gen
-from openmc_data.generate import generate_jeff33 as legacy
from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements
@@ -110,21 +109,14 @@ def test_release_particle_selection(tmp_path, processing, release, count, partic
assert not list(tmp_path.glob('.jeff-generate-*'))
-def test_defaults_and_legacy_import_safe(monkeypatch):
+def test_defaults_and_import_safe(monkeypatch):
monkeypatch.setattr(sys, 'argv', ['program', '--not-a-generator-option'])
importlib.reload(gen)
- importlib.reload(legacy)
args = gen._parser().parse_args([])
assert args.release == '4.0' and not args.use_tmpdir
+ assert args.destination is None and args.libver == 'earliest'
assert args.temperatures == [250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0]
assert args.particles == ['neutron', 'thermal', 'photon']
- args = gen._parser(legacy=True).parse_args([])
- assert args.release == '3.3' and args.use_tmpdir
- assert args.destination == Path('jeff33_hdf5')
- main = MagicMock()
- monkeypatch.setattr(legacy, '_main', main)
- legacy.main(['--jobs', '2'])
- main.assert_called_once_with(['--jobs', '2'], legacy=True)
@pytest.mark.parametrize('argv', [
@@ -253,14 +245,15 @@ def prepare(details, archives, directory, args):
assert Path.cwd() == tmp_path
-def test_legacy_offline_tmpdir_uses_persistent_inputs(tmp_path, processing):
- inputs(tmp_path / 'jeff-3.3-endf', '3.3')
+@pytest.mark.parametrize('release', ['3.3', '4.0'])
+def test_offline_tmpdir_uses_persistent_inputs(tmp_path, processing, release):
+ inputs(tmp_path / f'jeff-{release}-endf', release)
cwd = Path.cwd()
with pytest.warns(UserWarning, match='ignored'):
- legacy.main(['-p', 'photon', '--no-download', '--no-extract', '--jobs', '1', '--cleanup'])
+ gen.main(offline(release, ['photon']) + ['--use-tmpdir', '--cleanup'])
assert Path.cwd() == cwd
- assert (tmp_path / 'jeff33_hdf5/cross_sections.xml').is_file()
- assert (tmp_path / 'jeff-3.3-endf/photon').exists()
+ assert (tmp_path / f'jeff-{release}-hdf5/cross_sections.xml').is_file()
+ assert (tmp_path / f'jeff-{release}-endf/photon').exists()
@pytest.mark.parametrize('download_enabled', [False, True])
From ab2dd789cdd2b6bb7b0eb0f5d95f0b225f412148 Mon Sep 17 00:00:00 2001
From: shimwell <8583900+shimwell@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:22:11 +0200
Subject: [PATCH 4/5] Defer expanded JEFF generation tests
---
README.md | 2 -
pyproject.toml | 3 +-
tests/test_generate_jeff.py | 447 ------------------------------------
tests/test_urls.py | 58 -----
tests/test_utils.py | 80 -------
5 files changed, 1 insertion(+), 589 deletions(-)
delete mode 100644 tests/test_generate_jeff.py
diff --git a/README.md b/README.md
index 1b19821..b9b1962 100644
--- a/README.md
+++ b/README.md
@@ -139,8 +139,6 @@ hold original archive basenames and nested inputs; 3.3 replacements use
with a warning for reused inputs or explicit cache roots.
See `generate_jeff --help` for all options.
-Tests use real HDF5/XML with mocked NJOY; opt-in bounded real-data samples require
-`OPENMC_JEFF_INTEGRATION=1` and NJOY. Live URL checks run separately.
### Download cross sections
diff --git a/pyproject.toml b/pyproject.toml
index 2952aa9..f9a8c4a 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -34,8 +34,7 @@ write_to = "src/openmc_data/_version.py"
[project.optional-dependencies]
tests = [
"pytest",
- "requests",
- "stream-unzip",
+ "requests"
]
[tool.pytest.ini_options]
diff --git a/tests/test_generate_jeff.py b/tests/test_generate_jeff.py
deleted file mode 100644
index bda476c..0000000
--- a/tests/test_generate_jeff.py
+++ /dev/null
@@ -1,447 +0,0 @@
-"""Offline orchestration tests and opt-in real NJOY sample tests.
-
-Offline tests use real HDF5/XML with mocked NJOY. Integration tests require
-OPENMC_JEFF_INTEGRATION=1 and NJOY on PATH, and stream bounded archive samples
-rather than saving complete libraries. They are not full-library qualification.
-"""
-
-from contextlib import nullcontext
-import hashlib
-import importlib
-import os
-from pathlib import Path
-import sys
-import tarfile
-import tempfile
-from types import SimpleNamespace
-from unittest.mock import MagicMock
-import zipfile
-
-import h5py
-import numpy as np
-import openmc.data
-import pytest
-import requests
-from stream_unzip import stream_unzip
-
-from openmc_data.generate import generate_jeff as gen
-from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements
-
-
-@pytest.fixture(autouse=True)
-def temporary_workdir(tmp_path, monkeypatch):
- """Keep working files and temporary staging/NJOY files inside tmp_path."""
- monkeypatch.chdir(tmp_path)
- monkeypatch.setattr(tempfile, 'tempdir', str(tmp_path))
- monkeypatch.setenv('TMPDIR', str(tmp_path))
-
-
-def put(path, content='input'):
- path.parent.mkdir(parents=True, exist_ok=True)
- path.write_text(content)
- return path
-
-
-def hdf5(directory, name, particle):
- with h5py.File(directory / f'{name}.h5', 'w') as f:
- f.attrs['filetype'] = np.bytes_('data_' + particle)
- f.create_group(name)
-
-
-def inputs(root, release):
- pairs = jeff_endf_details[release]['thermal']['pairs']
- for i, name in enumerate(sorted({n for n, _ in pairs})):
- put(root / 'neutron' / 'nested' / name, f'N{i}')
- for i, (_, name) in enumerate(pairs):
- put(root / 'thermal' / 'nested' / name, f'c_material{i}')
- for prefix in ['photoat', 'atom']:
- for z, symbol in [(1, 'H'), (8, 'O')]:
- put(root / 'photon' / prefix / f'{prefix}-{z:03}_{symbol}_000.endf', symbol)
-
-
-@pytest.fixture
-def processing(tmp_path, monkeypatch):
- calls = []
- def neutron(path, output, libver, temperatures):
- calls.append(('neutron', path, libver, temperatures))
- hdf5(output, path.read_text(), 'neutron')
- def thermal(path, tsl, output, libver):
- calls.append(('thermal', path, tsl, libver))
- hdf5(output, tsl.read_text(), 'thermal')
- def photon(photo, atom):
- assert photo.read_text() == atom.read_text()
- calls.append(('photon', photo, atom))
- name = photo.read_text()
- def export(path, mode, libver):
- assert mode == 'w'
- hdf5(path.parent, name, 'photon')
- return SimpleNamespace(name=name, export_to_hdf5=export)
- monkeypatch.setattr(gen, 'process_neutron', neutron)
- monkeypatch.setattr(gen, 'process_thermal', thermal)
- monkeypatch.setattr(openmc.data.IncidentPhoton, 'from_endf', photon)
- monkeypatch.setattr(gen, 'download', MagicMock(side_effect=AssertionError('Unexpected network access')))
- return calls
-
-
-def offline(release, particles):
- return ['-r', release, '-p', *particles, '--jobs', '1', '--no-download', '--no-extract']
-
-
-@pytest.mark.parametrize('release,count', [('3.3', 20), ('4.0', 128)])
-@pytest.mark.parametrize('particles', [('neutron',), ('thermal',), ('photon',), ('neutron', 'thermal', 'photon')])
-def test_release_particle_selection(tmp_path, processing, release, count, particles):
- root = tmp_path / f'jeff-{release}-endf'
- inputs(root, release)
- destination = tmp_path / f'jeff-{release}-hdf5'
- put(destination / 'neutron' / 'stale.h5', 'not HDF5')
- gen.main(offline(release, particles) + ['--temperatures', '300', '700', '--libver', 'latest'])
- library = openmc.data.DataLibrary.from_xml(destination / 'cross_sections.xml')
- assert {entry['type'] for entry in library} == set(particles)
- assert all(Path(entry['path']).is_file() for entry in library)
- assert all('stale' not in entry['path'] and '.jeff-' not in entry['path'] for entry in library)
- assert (destination / 'neutron' / 'stale.h5').read_text() == 'not HDF5'
- assert len([c for c in processing if c[0] == 'thermal']) == (count if 'thermal' in particles else 0)
- for call in processing:
- if call[0] == 'neutron':
- assert call[2:] == ('latest', [300.0, 700.0])
- if call[0] == 'thermal':
- assert len(call) == 4 # no neutron temperature override
- assert not list(tmp_path.glob('.jeff-generate-*'))
-
-
-def test_defaults_and_import_safe(monkeypatch):
- monkeypatch.setattr(sys, 'argv', ['program', '--not-a-generator-option'])
- importlib.reload(gen)
- args = gen._parser().parse_args([])
- assert args.release == '4.0' and not args.use_tmpdir
- assert args.destination is None and args.libver == 'earliest'
- assert args.temperatures == [250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0]
- assert args.particles == ['neutron', 'thermal', 'photon']
-
-
-@pytest.mark.parametrize('argv', [
- ['-r', '3.2'], ['--jobs', '0'], ['--jobs', '-1'], ['--temperatures', '0'],
- ['--temperatures', '-5'], ['--temperatures', 'nan'], ['--temperatures', 'inf'],
- ['--temperatures', '293.6', '294'], ['-p', 'electron'],
- ['-d', 'cache/out', '--endf-dir', 'cache'],
- ['--endf-dir', 'cache', '--download-dir', 'cache/download'],
-])
-def test_invalid_arguments(argv, tmp_path, monkeypatch):
- with pytest.raises(SystemExit) as error:
- gen.main(argv)
- assert error.value.code == 2
- assert not list(tmp_path.iterdir())
-
-
-@pytest.mark.parametrize('release', ['3.3', '4.0'])
-def test_thermal_download_dependency(tmp_path, processing, monkeypatch, release):
- inputs(tmp_path / f'jeff-{release}-endf', release)
- prepare = MagicMock()
- monkeypatch.setattr(gen, '_prepare', prepare)
- gen.main(['-r', release, '-p', 'thermal', '--jobs', '1'])
- assert [call.args[2].name for call in prepare.call_args_list] == ['neutron', 'thermal']
- assert {c[0] for c in processing} == {'thermal'}
- assert not (tmp_path / f'jeff-{release}-hdf5/neutron').exists()
-
-
-@pytest.mark.parametrize('problem', ['empty', 'missing_neutron', 'missing_thermal', 'duplicate'])
-def test_missing_and_ambiguous_inputs(tmp_path, processing, problem):
- root = tmp_path / 'jeff-4.0-endf'
- if problem != 'empty':
- inputs(root, '4.0')
- n, t = jeff_endf_details['4.0']['thermal']['pairs'][0]
- if problem == 'missing_neutron':
- (root / 'neutron/nested' / n).unlink()
- elif problem == 'missing_thermal':
- (root / 'thermal/nested' / t).unlink()
- else:
- put(root / 'neutron/duplicate' / n)
- with pytest.raises((FileNotFoundError, ValueError)):
- gen.main(offline('4.0', ['thermal']))
- assert not processing
- assert not (tmp_path / 'jeff-4.0-hdf5/cross_sections.xml').exists()
-
-
-@pytest.mark.parametrize('problem', ['missing', 'mismatch', 'duplicate'])
-def test_photon_pair_validation(tmp_path, processing, problem):
- root = tmp_path / 'jeff-4.0-endf'
- inputs(root, '4.0')
- atom = root / 'photon/atom/atom-001_H_000.endf'
- if problem == 'missing':
- atom.unlink()
- elif problem == 'mismatch':
- atom.rename(atom.with_name('atom-002_He_000.endf'))
- else:
- put(root / 'photon/duplicate' / atom.name, 'H')
- with pytest.raises(ValueError):
- gen.main(offline('4.0', ['photon']))
- assert not processing
-
-
-@pytest.mark.parametrize('existing', [False, True])
-@pytest.mark.parametrize('failure', ['worker', 'collision', 'empty', 'invalid_hdf5'])
-def test_processing_failure_never_publishes(tmp_path, processing, monkeypatch, existing, failure):
- root = tmp_path / 'jeff-4.0-endf'
- inputs(root, '4.0')
- dest = tmp_path / 'jeff-4.0-hdf5'
- if existing:
- put(dest / 'cross_sections.xml', 'old manifest')
- put(dest / 'neutron/N0.h5', 'old data')
- def broken(path, output, libver, temperatures):
- if failure == 'worker':
- raise ValueError('NJOY failed')
- if failure == 'collision':
- hdf5(output, 'same', 'neutron')
- if failure == 'invalid_hdf5':
- put(output / 'bad.h5', 'invalid')
- monkeypatch.setattr(gen, 'process_neutron', broken)
- with pytest.raises((RuntimeError, ValueError, OSError)):
- gen.main(offline('4.0', ['neutron']) + ['--cleanup'])
- if existing:
- assert (dest / 'cross_sections.xml').read_text() == 'old manifest'
- assert (dest / 'neutron/N0.h5').read_text() == 'old data'
- else:
- assert not dest.exists()
- assert root.is_dir()
- assert not list(tmp_path.glob('.jeff-generate-*'))
-
-
-def test_pool_exception_is_retrieved(tmp_path, processing, monkeypatch):
- inputs(tmp_path / 'jeff-4.0-endf', '4.0')
- pool = MagicMock()
- pool.__enter__.return_value = pool
- pool.apply_async.return_value.get.side_effect = RuntimeError('remote NJOY failure')
- factory = MagicMock(return_value=pool)
- monkeypatch.setattr(gen, 'Pool', factory)
- with pytest.raises(RuntimeError, match='remote NJOY failure'):
- gen.main(offline('4.0', ['neutron']) + ['--jobs', '2'])
- factory.assert_called_once_with(processes=2)
- pool.apply_async.return_value.get.assert_called_once()
- pool.apply_async.return_value.wait.assert_not_called()
- assert not (tmp_path / 'jeff-4.0-hdf5/cross_sections.xml').exists()
-
-
-@pytest.mark.parametrize('temporary', [False, True])
-@pytest.mark.parametrize('failure', [False, True])
-def test_input_cleanup(tmp_path, processing, monkeypatch, temporary, failure):
- root = tmp_path / 'jeff-4.0-endf'
- inputs(root, '4.0')
- archive = put(tmp_path / 'jeff-4.0-download/photon/data.zip')
- other = put(tmp_path / 'jeff-4.0-download/neutron/keep.zip')
- roots = []
- def prepare(details, archives, directory, args):
- roots.append(directory.parent)
- inputs(directory.parent, '4.0')
- monkeypatch.setattr(gen, '_prepare', prepare)
- if failure:
- monkeypatch.setattr(gen, '_generate', MagicMock(side_effect=RuntimeError('failed')))
- with pytest.raises(RuntimeError) if failure else nullcontext():
- gen.main(['-p', 'photon', '--jobs', '1', '--use-tmpdir' if temporary else '--cleanup'])
- assert roots
- assert all(path.exists() is not temporary for path in roots)
- assert archive.exists() is (temporary or failure)
- assert (root / 'photon').exists() is (temporary or failure)
- assert other.exists() and (root / 'neutron').exists()
- assert Path.cwd() == tmp_path
-
-
-@pytest.mark.parametrize('release', ['3.3', '4.0'])
-def test_offline_tmpdir_uses_persistent_inputs(tmp_path, processing, release):
- inputs(tmp_path / f'jeff-{release}-endf', release)
- cwd = Path.cwd()
- with pytest.warns(UserWarning, match='ignored'):
- gen.main(offline(release, ['photon']) + ['--use-tmpdir', '--cleanup'])
- assert Path.cwd() == cwd
- assert (tmp_path / f'jeff-{release}-hdf5/cross_sections.xml').is_file()
- assert (tmp_path / f'jeff-{release}-endf/photon').exists()
-
-
-@pytest.mark.parametrize('download_enabled', [False, True])
-def test_archive_cache_checksum_and_fresh_extraction(tmp_path, monkeypatch, download_enabled):
- archives = tmp_path / 'archives'
- archives.mkdir()
- archive = archives / 'library.zip'
- with zipfile.ZipFile(archive, 'w') as z:
- z.writestr('nested/H1.jeff', 'H1')
- checksum = hashlib.md5(archive.read_bytes()).hexdigest()
- details = {'base_url': 'https://example.invalid/', 'compressed_files': ['library.zip'],
- 'checksums': [checksum], 'url_suffix': '/content'}
- args = SimpleNamespace(download=download_enabled, extract=True)
- network = MagicMock(side_effect=AssertionError('Valid cache must not access network'))
- monkeypatch.setattr(gen, 'download', network)
- target = tmp_path / 'inputs'
- put(target / 'stale.jeff')
- gen._prepare(details, archives, target, args)
- assert (target / 'nested/H1.jeff').read_text() == 'H1'
- assert not (target / 'stale.jeff').exists()
- network.assert_not_called()
- archive.write_bytes(b'corrupt')
- if not download_enabled:
- with pytest.raises(OSError, match='MD5'):
- gen._prepare(details, archives, target, args)
- else:
- with pytest.raises(AssertionError, match='network'):
- gen._prepare(details, archives, target, args)
- assert network.call_args.kwargs['output_filename'] == 'library.zip'
- assert network.call_args.args[0].endswith('library.zip/content')
- assert (target / 'nested/H1.jeff').is_file()
-
-
-def test_missing_archive_is_actionable(tmp_path):
- with pytest.raises(FileNotFoundError, match='enable --download'):
- gen._prepare(jeff_endf_details['4.0']['neutron'], tmp_path / 'archives',
- tmp_path / 'inputs', SimpleNamespace(download=False, extract=True))
-
-
-def test_replacements_do_not_rename_originals(tmp_path, processing, monkeypatch):
- inputs(tmp_path / 'jeff-3.3-endf', '3.3')
- for name, details in jeff33_replacements.items():
- put(tmp_path / 'jeff-3.3-endf/neutron/nested' / name, 'original')
- put(tmp_path / 'jeff-3.3-endf/replacements' / Path(name).stem / 'evaluation.dat', name[:1])
- targets = {'6': (6, 13), '8': (8, 17)}
- def evaluation(path):
- z, a = targets[path.read_text()]
- return SimpleNamespace(target={'atomic_number': z, 'mass_number': a, 'isomeric_state': 0})
- monkeypatch.setattr(openmc.data.endf, 'Evaluation', evaluation)
- gen.main(offline('3.3', ['neutron']))
- assert len([c for c in processing if c[1].suffix == '.dat']) == 2
- for name in jeff33_replacements:
- assert (tmp_path / 'jeff-3.3-endf/neutron/nested' / name).read_text() == 'original'
- assert (tmp_path / 'jeff-3.3-endf/replacements' / Path(name).stem / 'evaluation.dat').exists()
-
-
-@pytest.mark.parametrize('problem', ['missing', 'multiple', 'wrong_target', 'metastable'])
-def test_replacement_validation(tmp_path, monkeypatch, problem):
- details = jeff33_replacements['6-C-13g.jeff33']
- if problem != 'missing':
- put(tmp_path / 'evaluation.dat')
- if problem == 'multiple':
- put(tmp_path / 'other.dat')
- target = {'atomic_number': 8 if problem == 'wrong_target' else 6,
- 'mass_number': 13, 'isomeric_state': int(problem == 'metastable')}
- monkeypatch.setattr(openmc.data.endf, 'Evaluation', lambda _: SimpleNamespace(target=target))
- with pytest.raises(ValueError):
- gen._replacement(details, tmp_path)
-
-
-def test_publication_failure_removes_manifest(tmp_path, processing, monkeypatch):
- inputs(tmp_path / 'jeff-4.0-endf', '4.0')
- dest = tmp_path / 'jeff-4.0-hdf5'
- put(dest / 'cross_sections.xml', 'old manifest')
- replace = Path.replace
- def fail(path, target):
- if Path(target).parent == dest / 'photon':
- raise OSError('disk failure')
- return replace(path, target)
- monkeypatch.setattr(Path, 'replace', fail)
- with pytest.raises(OSError, match='disk failure'):
- gen.main(offline('4.0', ['photon']))
- assert not (dest / 'cross_sections.xml').exists()
-
-
-def test_metadata_invariants():
- assert set(jeff_endf_details) == {'3.3', '4.0'}
- for release, count in [('3.3', 20), ('4.0', 128)]:
- details = jeff_endf_details[release]
- assert set(details) == {'neutron', 'thermal', 'photon'}
- pairs = details['thermal']['pairs']
- assert len(pairs) == len({t for n, t in pairs}) == count
- assert all(n.endswith(details['neutron']['pattern'][1:]) for n, _ in pairs)
- for group in details.values():
- assert len(group['compressed_files']) == len(group['checksums'])
- assert all(len(checksum) == 32 for checksum in group['checksums'])
- assert ('VIII.0' if release == '3.3' else 'VIII.1') in details['photon']['compressed_files'][0]
- assert ('n_8-O-016g.jeff', 'tsl_O_D20.jeff') in jeff_endf_details['4.0']['thermal']['pairs']
- assert len(jeff33_replacements) == 2
-
-
-# Opt-in real NJOY samples, separate from the offline orchestration above.
-class BoundedStream:
- def __init__(self, raw, limit):
- self.raw, self.limit, self.count = raw, limit, 0
-
- def read(self, size):
- if size < 0 or self.count + size > self.limit:
- raise RuntimeError('Archive sample exceeded its byte budget')
- data = self.raw.read(size)
- self.count += len(data)
- return data
-
-
-def sample(details, wanted, destination, budget):
- """Extract named samples while streaming; close immediately after finding them."""
- name = details['compressed_files'][0]
- url = details['base_url'] + name + details.get('url_suffix', '')
- missing = set(wanted)
- destination.mkdir(parents=True, exist_ok=True)
- with requests.get(url, stream=True, headers={'Accept-Encoding': 'identity'}, timeout=(10, 60)) as r:
- r.raise_for_status()
- stream = BoundedStream(r.raw, budget)
- if name.endswith('.zip'):
- # Delegate ZIP64, decompression, CRC and truncation checks to the reader.
- for member, _, chunks in stream_unzip(iter(lambda: stream.read(65536), b'')):
- target = Path(member.decode()).name
- if target in missing:
- with (destination / target).open('wb') as output:
- for chunk in chunks:
- output.write(chunk)
- missing.remove(target)
- if not missing:
- break
- else:
- for _ in chunks:
- pass
- else:
- with tarfile.open(fileobj=stream, mode='r|gz') as archive:
- for member in archive:
- target = Path(member.name).name
- if target in missing and member.isfile():
- with archive.extractfile(member) as source:
- (destination / target).write_bytes(source.read())
- missing.remove(target)
- if not missing:
- break
- assert not missing, missing
- print(f'Sampled {name}: {stream.count} compressed bytes')
-
-
-@pytest.mark.network
-@pytest.mark.integration
-@pytest.mark.skipif(os.environ.get('OPENMC_JEFF_INTEGRATION') != '1',
- reason='set OPENMC_JEFF_INTEGRATION=1 for opt-in network/NJOY sample validation')
-@pytest.mark.parametrize('release,particle,n,t', [
- ('3.3', 'neutron', '1-H-1g.jeff33', None),
- ('4.0', 'neutron', 'n_1-H-001g.jeff', None),
- ('3.3', 'thermal', '1-H-1g.jeff33', 'tsl-HinH2O.jeff33'),
- ('4.0', 'thermal', 'n_3-Li-007g.jeff', 'tsl_7Li_7LiD.jeff'),
- ('3.3', 'photon', None, None), ('4.0', 'photon', None, None),
-])
-def test_real_sample(tmp_path, release, particle, n, t):
- root = tmp_path / f'jeff-{release}-endf'
- destination = tmp_path / f'jeff-{release}-hdf5'
- details = jeff_endf_details[release]
- if n:
- sample(details['neutron'], [n], root / 'neutron', 128 * 1024**2)
- if t:
- sample(details['thermal'], [t], root / 'thermal', 40 * 1024**2)
- args = gen._parser().parse_args(['--jobs', '1', '--temperatures', '700'])
- gen._generate([('thermal', (root / 'neutron' / n, root / 'thermal' / t))], destination, args)
- else:
- if particle == 'photon':
- for archive, prefix in zip(details['photon']['compressed_files'], ['photoat', 'atom']):
- sample(dict(details['photon'], compressed_files=[archive]),
- [f'{prefix}-001_H_000.endf'], root / 'photon', 32 * 1024**2)
- # Real neutron workers exercise child-process import safety and error transport.
- gen.main(offline(release, [particle]) + ['--temperatures', '293.6',
- '--jobs', '2' if n else '1'])
- outputs = list((destination / particle).glob('*.h5'))
- assert len(outputs) == 1
- if particle == 'neutron':
- data = openmc.data.IncidentNeutron.from_hdf5(outputs[0])
- assert data.name == 'H1' and '294K' in data.temperatures
- elif particle == 'thermal':
- data = openmc.data.ThermalScattering.from_hdf5(outputs[0])
- assert len(data.temperatures) > 1 and data.temperatures != ['700K']
- else:
- assert openmc.data.IncidentPhoton.from_hdf5(outputs[0]).atomic_number == 1
- assert len(openmc.data.DataLibrary.from_xml(destination / 'cross_sections.xml')) == 1
\ No newline at end of file
diff --git a/tests/test_urls.py b/tests/test_urls.py
index 2801257..25002de 100644
--- a/tests/test_urls.py
+++ b/tests/test_urls.py
@@ -1,6 +1,4 @@
-import json
-import re
import time
from pathlib import Path
from unittest.mock import MagicMock
@@ -15,7 +13,6 @@
all_chain_release_details,
all_decay_release_details,
)
-from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements
# urls that serve a file kept in this repository, see check_url
@@ -273,58 +270,3 @@ def test_transient_failure_is_retried(http_response):
get.side_effect = [requests.Timeout('timed out'), response]
check_url('https://example.org/data')
assert get.call_count == 2
-
-
-def source_urls():
- groups = [(f'{release}-{particle}', details)
- for release, particles in jeff_endf_details.items()
- for particle, details in particles.items()]
- groups.extend((f'3.3-replacement-{name}', details) for name, details in jeff33_replacements.items())
- for label, details in groups:
- for name in details['compressed_files']:
- yield pytest.param(details['base_url'] + name + details.get('url_suffix', ''),
- id=f'{label}-{name}')
-
-
-@pytest.mark.parametrize('url', list(source_urls()))
-def test_generate_jeff_url_registry(url):
- validate_url(url)
-
-
-@pytest.mark.network
-@pytest.mark.parametrize('url', list(source_urls()))
-def test_generate_jeff_urls(url):
- check_url(url)
-
-
-def bounded_text(url):
- with requests.get(url, stream=True, headers={'Accept-Encoding': 'identity'}, timeout=(10, 60)) as r:
- r.raise_for_status()
- content = r.raw.read(2_000_001)
- assert len(content) <= 2_000_000, 'Metadata/preview unexpectedly large'
- return content.decode()
-
-
-@pytest.mark.network
-@pytest.mark.parametrize('release', ['3.3', '4.0'])
-@pytest.mark.parametrize('particle', ['neutron', 'thermal'])
-def test_nea_record_checksums(release, particle):
- details = jeff_endf_details[release][particle]
- record = json.loads(bounded_text(details['base_url'].rsplit('/files/', 1)[0]))
- for name, checksum in zip(details['compressed_files'], details['checksums']):
- entry = record['files']['entries'][name]
- assert entry['checksum'] == 'md5:' + checksum
- assert entry['links']['content'] == details['base_url'] + name + '/content'
-
-
-@pytest.mark.network
-@pytest.mark.parametrize('particle,column', [('neutron', 0), ('thermal', 1)])
-def test_jeff40_pairs_exist_in_archive_preview(particle, column):
- details = jeff_endf_details['4.0'][particle]
- record = details['base_url'].split('/records/')[1].split('/')[0]
- name = details['compressed_files'][0]
- text = bounded_text(f'https://data.oecd-nea.org/records/{record}/preview/{name}')
- members = re.findall(r'\s*([^<>]+\.jeff)', text)
- assert len(members) >= (593 if particle == 'neutron' else 128)
- required = {pair[column] for pair in jeff_endf_details['4.0']['thermal']['pairs']}
- assert not required - set(members)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 4ef8b4f..ddd8482 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -1,8 +1,4 @@
-import hashlib
-import io
-from pathlib import Path
import tarfile
-from unittest.mock import Mock
import pytest
@@ -33,79 +29,3 @@ def test_process_neutron_requires_openmc():
with pytest.raises(ModuleNotFoundError, match="openmc is required"):
openmc_data.process_neutron("dummy.endf", "out", "latest")
-
-
-class Response(io.BytesIO):
- def __init__(self, payload, length):
- super().__init__(payload)
- self.length = length
- self.read_sizes = []
-
- def read(self, size=-1):
- assert size == utils._BLOCK_SIZE, 'Download must use bounded reads'
- self.read_sizes.append(size)
- return super().read(size)
-
-
-@pytest.mark.parametrize('known_length', [False, True])
-@pytest.mark.parametrize('existing', [False, True])
-def test_download_streaming_length_and_cache(tmp_path, monkeypatch, capsys, known_length, existing):
- payload = b'x' * (2*utils._BLOCK_SIZE + 17)
- response = Response(payload, len(payload) if known_length else None)
- target = tmp_path / 'library.zip'
- if existing:
- target.write_bytes(b'o' * len(payload))
- urlopen = Mock(return_value=response)
- monkeypatch.setattr(utils, 'urlopen', urlopen)
- path = utils.download('https://example.invalid/library.zip/content', as_browser=True,
- output_path=tmp_path, output_filename='library.zip', timeout=10)
- assert path == target
- assert urlopen.call_args.args[0].get_header('User-agent') == 'Mozilla/5.0'
- assert urlopen.call_args.kwargs == {'timeout': 10}
- if existing and known_length:
- assert target.read_bytes() == b'o' * len(payload)
- assert not response.read_sizes
- else:
- assert target.read_bytes() == payload
- assert len(response.read_sizes) == 4
- output = capsys.readouterr().out
- if not known_length:
- assert 'bytes' in output and '%' not in output and 'Skipping' not in output
- assert response.closed
-
-
-@pytest.mark.parametrize('payload,length,cached,checksum_data,error', [
- (b'', None, None, None, None), (b'', 0, None, None, None),
- (b'new', 3, b'partial', None, None),
- (b'payload', None, None, b'payload', None),
- (b'payload', None, None, b'wrong', 'MD5 checksum'),
- (b'payload', 7, b'payload', b'payload', None),
- (b'payload', 7, b'invalid', b'payload', None),
- (b'partial', 100, None, None, 'Incomplete download'),
-])
-def test_download_validation(tmp_path, monkeypatch, payload, length, cached, checksum_data, error):
- target = tmp_path / 'file'
- if cached is not None:
- target.write_bytes(cached)
- response = Response(payload, length)
- monkeypatch.setattr(utils, 'urlopen', Mock(return_value=response))
- checksum = hashlib.md5(checksum_data).hexdigest() if checksum_data is not None else None
- if error:
- with pytest.raises(OSError, match=error):
- utils.download('https://example.invalid/file', checksum=checksum, output_path=tmp_path)
- else:
- assert utils.download('https://example.invalid/file', checksum=checksum,
- output_path=tmp_path).read_bytes() == payload
- assert bool(response.read_sizes) is not (cached == payload)
- assert response.closed
-
-
-def test_checksum_uses_bounded_reads(tmp_path, monkeypatch):
- payload = b'x' * (2*utils._BLOCK_SIZE + 17)
- stream = Response(payload, len(payload))
- opened = Mock(return_value=stream)
- monkeypatch.setattr(Path, 'open', opened)
- assert utils._file_md5(tmp_path / 'file') == hashlib.md5(payload).hexdigest()
- opened.assert_called_once_with('rb')
- assert len(stream.read_sizes) == 4
- assert stream.closed
From 83d07f50a1bd2e0957bbf73f85832660f1056953 Mon Sep 17 00:00:00 2001
From: shimwell <8583900+shimwell@users.noreply.github.com>
Date: Tue, 20 Oct 2026 17:52:17 +0200
Subject: [PATCH 5/5] Move JEFF generation guidance into CLI help
---
README.md | 35 -----------------
src/openmc_data/generate/generate_jeff.py | 48 +++++++++++++++++++----
2 files changed, 41 insertions(+), 42 deletions(-)
diff --git a/README.md b/README.md
index b9b1962..1243177 100644
--- a/README.md
+++ b/README.md
@@ -104,41 +104,6 @@ A few categories of scripts are available:
| generate_jeff | JEFF | 3.3
4.0 | NJOY |
| generate_tendl | TENDL | 2023
2025 | |
-#### JEFF generation
-
-`generate_jeff -r 4.0 -p neutron --temperatures 293.6 600 --jobs 4` processes
-ENDF with OpenMC and NJOY (`njoy` must be on `PATH`). Defaults: release 4.0,
-all three particles, CPU-count workers, `--libver earliest`, and neutron
-temperatures 250, 293.6, 600, 900, 1200 and 2500 K. Thermal temperatures are
-native; thermal-only runs fetch neutron inputs without generating neutron HDF5.
-
-NEA sources and published MD5 checksums are in [src/openmc_data/urls_jeff.py](src/openmc_data/urls_jeff.py).
-Photon supplements use ENDF/B-VIII.0 for JEFF 3.3 and VIII.1 for 4.0.
-The 20 legacy 3.3 and 128 [upstream 4.0 thermal pairs](https://github.com/openmc-dev/data/blob/master/generate_jeff.py)
-are supported; additional temperature-split PuO2/UO2/Zy4 evaluations are excluded.
-Full-library NJOY qualification has not been performed.
-
-For 3.3, C13/O17 use individual [IAEA TENDL-2019 replacements](https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/)
-without editing originals. ZIP CRCs and ENDF targets are checked (no published MD5).
-If IAEA blocks automated access, manually cache the archives; blocked downloads fail.
-
-Output goes into `--destination` (default `jeff-RELEASE-hdf5`) with particle
-subdirectories and relative XML paths. Isolated workers stage and validate outputs
-before publication; only current outputs are registered. Processing failures preserve
-existing data; publication removes the old manifest first and installs the new one last.
-
-Caches default to `jeff-RELEASE-download` and `jeff-RELEASE-endf`, overridable via
-`--download-dir`/`--endf-dir`. All three roots must be disjoint. Particle subdirectories
-hold original archive basenames and nested inputs; 3.3 replacements use
-`replacements/6-C-13g` and `replacements/8-O-17g` beneath each cache root.
-- `--no-download` extracts cached archives, checking published MD5s.
-- `--no-download --no-extract` processes local inputs. Neutron/photon subsets are
- allowed; thermal runs require every mapped pair.
-- `--cleanup` removes selected caches only after success and only for enabled stages.
-- `--use-tmpdir` uses temporary inputs for fresh downloads/extraction; it is ignored
- with a warning for reused inputs or explicit cache roots.
-
-See `generate_jeff --help` for all options.
### Download cross sections
diff --git a/src/openmc_data/generate/generate_jeff.py b/src/openmc_data/generate/generate_jeff.py
index 2abb24a..7b415cb 100644
--- a/src/openmc_data/generate/generate_jeff.py
+++ b/src/openmc_data/generate/generate_jeff.py
@@ -1,8 +1,36 @@
#!/usr/bin/env python3
"""Generate OpenMC HDF5 data from JEFF 3.3 or 4.0 ENDF evaluations using NJOY.
-Photons use ENDF/B-VIII.0 (JEFF 3.3) or VIII.1 (JEFF 4.0).
-Thermal scattering uses the native evaluation temperatures, not --temperatures.
+Requires njoy on PATH. Example:
+ generate_jeff -r 4.0 -p neutron --temperatures 293.6 600 --jobs 4
+
+Thermal scattering uses native temperatures, not --temperatures. Thermal-only
+runs fetch neutron inputs without generating neutron HDF5. Photon supplements
+use ENDF/B-VIII.0 for JEFF 3.3 and VIII.1 for 4.0.
+
+NEA sources and published MD5s are in openmc_data.urls_jeff. Supported thermal
+pairs are the 20 legacy 3.3 pairs and 128 upstream 4.0 pairs from:
+https://github.com/openmc-dev/data/blob/master/generate_jeff.py
+Additional temperature-split PuO2/UO2/Zy4 evaluations are excluded.
+Full-library NJOY qualification has not been performed.
+
+JEFF 3.3 C13/O17 use individual IAEA TENDL-2019 replacements without editing
+originals. ZIP CRCs and ENDF targets are checked (no published MD5). If IAEA
+blocks automated access, manually cache the archives; blocked downloads fail.
+
+Output uses particle subdirectories and relative XML paths. Isolated workers
+stage and validate outputs before publication; only current outputs are
+registered. Processing failures preserve existing data. Publication removes
+the old manifest first and installs the new one last.
+
+Destination, download and ENDF directories must not overlap. Cache particle
+subdirectories hold original archive basenames and nested inputs; 3.3
+replacements use replacements/6-C-13g and replacements/8-O-17g beneath each
+cache root. --no-download extracts cached archives, checking published MD5s.
+With --no-download --no-extract, local neutron/photon subsets are allowed;
+thermal runs require every mapped pair. --cleanup removes selected caches
+only after success and only for enabled stages. --use-tmpdir is ignored with
+a warning for reused inputs or explicit cache roots.
"""
import argparse
@@ -19,18 +47,24 @@
from openmc_data.utils import _file_md5
+class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter,
+ argparse.RawDescriptionHelpFormatter):
+ pass
+
+
def _parser():
- parser = argparse.ArgumentParser(description=__doc__,
- formatter_class=argparse.ArgumentDefaultsHelpFormatter)
- parser.add_argument('-r', '--release', choices=['3.3', '4.0'], default='4.0')
+ parser = argparse.ArgumentParser(description=__doc__, formatter_class=CustomFormatter)
+ parser.add_argument('-r', '--release', choices=['3.3', '4.0'], default='4.0',
+ help='JEFF release')
parser.add_argument('-d', '--destination', type=Path, help='Output directory (default: jeff-RELEASE-hdf5)')
parser.add_argument('-p', '--particles', nargs='+', choices=['neutron', 'thermal', 'photon'],
- default=['neutron', 'thermal', 'photon'])
+ default=['neutron', 'thermal', 'photon'], help='Data types to generate')
parser.add_argument('--temperatures', nargs='+', type=float,
default=[250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0],
help='Neutron temperatures in kelvin (thermal uses native temperatures)')
parser.add_argument('--jobs', type=int, default=None, help='NJOY worker count; default: CPU count')
- parser.add_argument('--libver', choices=['earliest', 'latest'], default='earliest')
+ parser.add_argument('--libver', choices=['earliest', 'latest'], default='earliest',
+ help='Output HDF5 version compatibility')
parser.add_argument('--download-dir', type=Path, help='Archive cache (default: jeff-RELEASE-download)')
parser.add_argument('--endf-dir', type=Path, help='Extracted inputs (default: jeff-RELEASE-endf)')
for flag, default, help_text in [