diff --git a/.github/workflows/test_package.yml b/.github/workflows/test_package.yml index 9fbbfa5..2cfab87 100644 --- a/.github/workflows/test_package.yml +++ b/.github/workflows/test_package.yml @@ -63,7 +63,7 @@ jobs: run: | generate_cendl --help generate_endf --help - generate_jeff33 --help + generate_jeff --help generate_jendl --help - name: Test download h5 scripts exist diff --git a/README.md b/README.md index 609d98d..1243177 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,7 @@ A few categories of scripts are available: | generate_endf | ENDF/B | VII.1
VIII.0
VIII.1 | NNDC | | generate_fendl | FENDL | 3.2c
3.2b
3.2a
3.2
3.1d
3.1a
3.0 | | | generate_jendl | JENDL | 4.0
5.0 | | +| generate_jeff | JEFF | 3.3
4.0 | NJOY | | generate_tendl | TENDL | 2023
2025 | | diff --git a/pyproject.toml b/pyproject.toml index 942fab3..f9a8c4a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ convert_tendl = "openmc_data.convert.convert_tendl:main" generate_cendl = "openmc_data.generate.generate_cendl:main" generate_endf = "openmc_data.generate.generate_endf:main" -generate_jeff33 = "openmc_data.generate.generate_jeff33:main" +generate_jeff = "openmc_data.generate.generate_jeff:main" generate_jendl = "openmc_data.generate.generate_jendl:main" generate_fendl = "openmc_data.generate.generate_fendl:main" generate_tendl = "openmc_data.generate.generate_tendl:main" diff --git a/src/openmc_data/generate/generate_jeff.py b/src/openmc_data/generate/generate_jeff.py new file mode 100644 index 0000000..7b415cb --- /dev/null +++ b/src/openmc_data/generate/generate_jeff.py @@ -0,0 +1,286 @@ +#!/usr/bin/env python3 +"""Generate OpenMC HDF5 data from JEFF 3.3 or 4.0 ENDF evaluations using NJOY. + +Requires njoy on PATH. Example: + generate_jeff -r 4.0 -p neutron --temperatures 293.6 600 --jobs 4 + +Thermal scattering uses native temperatures, not --temperatures. Thermal-only +runs fetch neutron inputs without generating neutron HDF5. Photon supplements +use ENDF/B-VIII.0 for JEFF 3.3 and VIII.1 for 4.0. + +NEA sources and published MD5s are in openmc_data.urls_jeff. Supported thermal +pairs are the 20 legacy 3.3 pairs and 128 upstream 4.0 pairs from: +https://github.com/openmc-dev/data/blob/master/generate_jeff.py +Additional temperature-split PuO2/UO2/Zy4 evaluations are excluded. +Full-library NJOY qualification has not been performed. + +JEFF 3.3 C13/O17 use individual IAEA TENDL-2019 replacements without editing +originals. ZIP CRCs and ENDF targets are checked (no published MD5). If IAEA +blocks automated access, manually cache the archives; blocked downloads fail. + +Output uses particle subdirectories and relative XML paths. Isolated workers +stage and validate outputs before publication; only current outputs are +registered. Processing failures preserve existing data. Publication removes +the old manifest first and installs the new one last. + +Destination, download and ENDF directories must not overlap. Cache particle +subdirectories hold original archive basenames and nested inputs; 3.3 +replacements use replacements/6-C-13g and replacements/8-O-17g beneath each +cache root. --no-download extracts cached archives, checking published MD5s. +With --no-download --no-extract, local neutron/photon subsets are allowed; +thermal runs require every mapped pair. --cleanup removes selected caches +only after success and only for enabled stages. --use-tmpdir is ignored with +a warning for reused inputs or explicit cache roots. +""" + +import argparse +from contextlib import ExitStack +import math +from multiprocessing import Pool +from pathlib import Path +import shutil +from tempfile import TemporaryDirectory +import warnings + +from openmc_data import download, extract, process_neutron, process_thermal, ProgressTracker +from openmc_data.urls_jeff import jeff_endf_details, jeff33_replacements +from openmc_data.utils import _file_md5 + + +class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter): + pass + + +def _parser(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=CustomFormatter) + parser.add_argument('-r', '--release', choices=['3.3', '4.0'], default='4.0', + help='JEFF release') + parser.add_argument('-d', '--destination', type=Path, help='Output directory (default: jeff-RELEASE-hdf5)') + parser.add_argument('-p', '--particles', nargs='+', choices=['neutron', 'thermal', 'photon'], + default=['neutron', 'thermal', 'photon'], help='Data types to generate') + parser.add_argument('--temperatures', nargs='+', type=float, + default=[250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0], + help='Neutron temperatures in kelvin (thermal uses native temperatures)') + parser.add_argument('--jobs', type=int, default=None, help='NJOY worker count; default: CPU count') + parser.add_argument('--libver', choices=['earliest', 'latest'], default='earliest', + help='Output HDF5 version compatibility') + parser.add_argument('--download-dir', type=Path, help='Archive cache (default: jeff-RELEASE-download)') + parser.add_argument('--endf-dir', type=Path, help='Extracted inputs (default: jeff-RELEASE-endf)') + for flag, default, help_text in [ + ('download', True, 'Download missing or invalid archives'), + ('extract', True, 'Extract archives into particle subdirectories'), + ('cleanup', False, 'Remove selected caches after success, only for enabled download/extract stages'), + ('use-tmpdir', False, 'Use temporary inputs for a fresh download and extraction'), + ]: + parser.add_argument('--' + flag, action='store_true', default=default, help=help_text) + parser.add_argument('--no-' + flag, dest=flag.replace('-', '_'), action='store_false') + return parser + + +def _prepare(details, archives, inputs, args): + """Keep caches reusable offline and check checksums even with --no-download.""" + for name, checksum in zip(details['compressed_files'], details['checksums']): + archive = archives / Path(name).name + if args.download and not (archive.is_file() and checksum + and _file_md5(archive) == checksum): + download(details['base_url'] + name + details.get('url_suffix', ''), + checksum=checksum, output_path=archives, + output_filename=archive.name, timeout=60) + if args.extract: + if not archive.is_file(): + raise FileNotFoundError(f'Missing archive {archive}; enable --download or supply the cache') + if checksum and _file_md5(archive) != checksum: + raise OSError(f'MD5 checksum does not match: {archive}') + if args.extract: + # A fresh extraction avoids discovering stale evaluations left by an old archive. + inputs.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix='.jeff-extract-', dir=inputs.parent) as tmp: + staging = Path(tmp) / 'inputs' + for name in details['compressed_files']: + extract(archives / Path(name).name, staging) + if inputs.exists(): + shutil.rmtree(inputs) + staging.replace(inputs) + + +def _index(directory, pattern): + result = {} + for path in sorted(directory.rglob(pattern)): + if path.is_file(): + if path.name in result: + raise ValueError(f'Ambiguous input {path.name}: {result[path.name]} and {path}') + result[path.name] = path + if not result: + raise FileNotFoundError(f'No {pattern} inputs in {directory}; enable --extract or supply ENDF files') + return result + + +def _required(index, name): + if name not in index: + raise FileNotFoundError(f'Missing required evaluation: {name}') + return index[name] + + +def _replacement(details, directory): + import openmc.data + + # IAEA ZIPs contain one ENDF evaluation; do not assume its member filename. + paths = [p for p in directory.rglob('*') if p.is_file()] + if len(paths) != 1: + raise ValueError(f'Expected one TENDL replacement evaluation in {directory}, found {len(paths)}') + evaluation = openmc.data.endf.Evaluation(paths[0]) + target = evaluation.target + if (target['atomic_number'], target['mass_number']) != details['target'] or target['isomeric_state']: + raise ValueError(f'Wrong TENDL replacement target in {paths[0]}') + return paths[0] + + +def _photon_pairs(directory): + photo = {name[len('photoat-'):]: path for name, path in _index(directory, 'photoat-*.endf').items()} + atom = {name[len('atom-'):]: path for name, path in _index(directory, 'atom-*.endf').items()} + if photo.keys() != atom.keys(): + raise ValueError(f'Unpaired photon evaluations: {sorted(photo.keys() ^ atom.keys())}') + return [(photo[key], atom[key]) for key in sorted(photo)] + + +def _process_task(particle, paths, output, libver, temperatures): + """Each task has its own output directory, so name collisions cannot overwrite data.""" + output.mkdir(parents=True) + try: + if particle == 'neutron': + process_neutron(paths[0], output, libver, temperatures) + elif particle == 'thermal': + process_thermal(*paths, output, libver) + else: + import openmc.data + data = openmc.data.IncidentPhoton.from_endf(*paths) + data.export_to_hdf5(output / f'{data.name}.h5', 'w', libver=libver) + files = list(output.glob('*.h5')) + if len(files) != 1 or not files[0].stat().st_size: + raise RuntimeError(f'Expected one nonempty HDF5 output in {output}') + return particle, files[0] + except Exception as exc: + raise RuntimeError(f'Failed processing {particle}: {paths}') from exc + + +def _generate(tasks, destination, args): + import openmc.data + + destination.parent.mkdir(parents=True, exist_ok=True) + with TemporaryDirectory(prefix='.jeff-generate-', dir=destination.parent) as tmp: + staging = Path(tmp) + work = [(particle, paths, staging / 'jobs' / str(i), args.libver, args.temperatures) + for i, (particle, paths) in enumerate(tasks)] + tracker = ProgressTracker(len(work), verb='Processed') + if args.jobs == 1: + results = [] + for task in work: + results.append(_process_task(*task)) + tracker.complete(task[1][-1].name) + else: + with Pool(processes=args.jobs) as pool: + pending = [pool.apply_async(_process_task, task, + callback=tracker.callback(task[1][-1].name)) for task in work] + # get(), not wait(): any worker error aborts before publication. + results = [result.get() for result in pending] + + library = openmc.data.DataLibrary() + outputs = {} + for particle, path in sorted(results, key=lambda item: (item[0], item[1].name)): + relative = Path(particle) / path.name + if relative in outputs: + raise ValueError(f'Output name collision: {relative}') + library.register_file(path) + outputs[relative] = path + manifest = staging / 'cross_sections.xml' + library.export_to_xml(manifest) + + # Make paths relative to the manifest, not the temporary staging directory. + # DataLibrary may otherwise add an absolute for single-particle runs. + from xml.etree import ElementTree as ET + tree = ET.parse(manifest) + root = tree.getroot() + directory = root.find('directory') + if directory is not None: + root.remove(directory) + for entry, relative in zip(root.findall('library'), outputs): + entry.set('path', relative.as_posix()) + tree.write(manifest, encoding='utf-8', xml_declaration=True) + + destination.mkdir(parents=True, exist_ok=True) + # Old data remain untouched on processing/validation failure. Once publishing + # starts, remove the old manifest so an I/O failure cannot advertise mixed data. + (destination / 'cross_sections.xml').unlink(missing_ok=True) + for relative, path in outputs.items(): + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + path.replace(target) + manifest.replace(destination / 'cross_sections.xml') + + +def main(argv=None): + parser = _parser() + args = parser.parse_args(argv) + if args.jobs is not None and args.jobs < 1: + parser.error('--jobs must be positive') + if any(not math.isfinite(t) or t <= 0 for t in args.temperatures): + parser.error('--temperatures must be finite and positive') + if len({round(t) for t in args.temperatures}) != len(args.temperatures): + parser.error('--temperatures must have distinct rounded kelvin values (OpenMC HDF5 labels)') + particles = set(args.particles) + required = [p for p in ('neutron', 'thermal', 'photon') + if p in particles or (p == 'neutron' and 'thermal' in particles)] + details = jeff_endf_details[args.release] + destination = (args.destination or Path(f'jeff-{args.release}-hdf5')).resolve() + archives = (args.download_dir or Path(f'jeff-{args.release}-download')).resolve() + inputs = (args.endf_dir or Path(f'jeff-{args.release}-endf')).resolve() + for a, b in ((destination, archives), (destination, inputs), (archives, inputs)): + if a == b or a in b.parents or b in a.parents: + parser.error('Destination, download and ENDF directories must not overlap') + + with ExitStack() as stack: + if args.use_tmpdir: + if args.download and args.extract and not args.download_dir and not args.endf_dir: + temporary = Path(stack.enter_context(TemporaryDirectory(prefix='jeff-inputs-'))) + archives, inputs = temporary / 'download', temporary / 'endf' + else: + warnings.warn('--use-tmpdir ignored when reusing local inputs or explicit cache directories') + prepared = [] + for particle in required: + _prepare(details[particle], archives / particle, inputs / particle, args) + prepared.append(particle) + + tasks = [] + if 'neutron' in required: + neutrons = _index(inputs / 'neutron', details['neutron']['pattern']) + if 'neutron' in particles: + selected = dict(neutrons) + if args.release == '3.3': + for name, replacement in jeff33_replacements.items(): + if name in selected: + group = 'replacements/' + Path(name).stem + _prepare(replacement, archives / group, inputs / group, args) + prepared.append(group) + selected[name] = _replacement(replacement, inputs / group) + print(f'Using TENDL-2019 replacement for {name}') + tasks.extend(('neutron', (path,)) for path in selected.values()) + if 'thermal' in particles: + thermal = _index(inputs / 'thermal', details['neutron']['pattern']) + tasks.extend(('thermal', (_required(neutrons, n), _required(thermal, t))) + for n, t in details['thermal']['pairs']) + if 'photon' in particles: + tasks.extend(('photon', pair) for pair in _photon_pairs(inputs / 'photon')) + if not tasks: + raise ValueError('No evaluations selected for generation') + _generate(tasks, destination, args) + if args.cleanup: + for group in prepared: + for root, enabled in ((archives, args.download), (inputs, args.extract)): + if enabled and (root / group).exists(): + shutil.rmtree(root / group) + print(f'Wrote {destination / "cross_sections.xml"}') + + +if __name__ == '__main__': + main() \ No newline at end of file diff --git a/src/openmc_data/generate/generate_jeff33.py b/src/openmc_data/generate/generate_jeff33.py deleted file mode 100755 index 5828c5c..0000000 --- a/src/openmc_data/generate/generate_jeff33.py +++ /dev/null @@ -1,216 +0,0 @@ -#!/usr/bin/env python3 - -""" -Generate JEFF 3.3 HDF5 library for use in OpenMC by first processing ENDF files -using NJOY. The resulting library will contain incident neutron, photoatomic, -and thermal scattering data. Note that JEFF doesn't distribute photoatomic or -atomic relaxation data so these are obtained from ENDF/B-VIII.0. - -""" - -import argparse -import os -import sys -import tarfile -import tempfile -import zipfile -from multiprocessing import Pool -from pathlib import Path -from urllib.parse import urljoin - -import openmc.data - -from openmc_data import download, process_neutron, process_thermal, ProgressTracker - -# Make sure Python version is sufficient -assert sys.version_info >= (3, 6), "Python 3.6+ is required" - - -class CustomFormatter(argparse.ArgumentDefaultsHelpFormatter, - argparse.RawDescriptionHelpFormatter): - pass - - -parser = argparse.ArgumentParser( - description=__doc__, - formatter_class=CustomFormatter -) -parser.add_argument('-d', '--destination', type=Path, default=Path('jeff33_hdf5'), - help='Directory to create new library in') -parser.add_argument('--libver', choices=['earliest', 'latest'], - default='earliest', help="Output HDF5 versioning. Use " - "'earliest' for backwards compatibility or 'latest' for " - "performance") -parser.add_argument('--download', action='store_true', - help='Download files') -parser.add_argument('--no-download', dest='download', action='store_false', - help='Do not download files') -parser.add_argument('--use-tmpdir', dest='tmpdir', action='store_true', - help='Use temporary directory while processing') -parser.add_argument('--no-use-tmpdir', dest='tmpdir', action='store_false', - help='Do not use temporary directory while processing') -parser.add_argument('--extract', action='store_true', - help='Extract zip files') -parser.add_argument('--no-extract', dest='extract', action='store_false', - help='Do not extract zip files') -parser.add_argument('--temperatures', type=float, - default=[250.0, 293.6, 600.0, 900.0, 1200.0, 2500.0], - help="Temperatures in Kelvin", nargs='+') -parser.set_defaults(download=True, extract=True, tmpdir=True) -args = parser.parse_args() - - -def sort_key(path): - if path.name.startswith('c_'): - # Ensure that thermal scattering gets sorted after neutron data - return (1000, path) - else: - return openmc.data.zam(path.stem) - - -def main(): - - base_endf = 'https://www.nndc.bnl.gov/endf-b8.0/zips/' - base_jeff = 'http://www.oecd-nea.org/dbdata/jeff/jeff33/downloads/' - files = [ - (base_jeff, 'JEFF33-n.tgz', 'e540bbf95179257280c61acfa75c83de'), - (base_jeff, 'JEFF33-tsl.tgz', '82a6df4cb802aa4a09b95309f7861c54'), - (base_endf, 'ENDF-B-VIII.0_photoat.zip', 'd49f5b54be278862e1ce742ccd94f5c0'), - (base_endf, 'ENDF-B-VIII.0_atomic_relax.zip', 'e04d50098cb2a7e4fe404ec4071611cc'), - ] - - # TODO: these individual TENDL neutron file endpoints return 404 on the - # new tendl.imperial.ac.uk site. Re-enable once the files are available. - tendl_files = [ - # 'https://tendl.imperial.ac.uk/tendl_2019/neutron_file/C/C013/lib/endf/n-C013.tendl', - # 'https://tendl.imperial.ac.uk/tendl_2019/neutron_file/O/O017/lib/endf/n-O017.tendl', - ] - - neutron_dir = Path('endf6') - thermal_dir = Path('JEFF33-tsl') - photoat_dir = Path('ENDF-B-VIII.0_photoat') - atomic_dir = Path('ENDF-B-VIII.0_atomic_relax') - thermal_paths = [ - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinCaH2.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinCH2.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinH2O.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinIce.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinMesitylene-PhaseII.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinOrthoH.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinParaH.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinToluene.jeff33'), - (neutron_dir / '1-H-1g.jeff33', thermal_dir / 'tsl-HinZrH.jeff33'), - (neutron_dir / '1-H-2g.jeff33', thermal_dir / 'tsl-DinD2O.jeff33'), - (neutron_dir / '1-H-2g.jeff33', thermal_dir / 'tsl-DinOrthoD.jeff33'), - (neutron_dir / '1-H-2g.jeff33', thermal_dir / 'tsl-DinParaD.jeff33'), - (neutron_dir / '4-Be-9g.jeff33', thermal_dir / 'tsl-Be.jeff33'), - (neutron_dir / '6-C-0g.jeff33', thermal_dir / 'tsl-Graphite.jeff33'), - (neutron_dir / '8-O-16g.jeff33', thermal_dir / 'tsl-O16inAl2O3.jeff33'), - (neutron_dir / '8-O-16g.jeff33', thermal_dir / 'tsl-OinD2O.jeff33'), - (neutron_dir / '12-Mg-24g.jeff33', thermal_dir / 'tsl-Mg.jeff33'), - (neutron_dir / '13-Al-27g.jeff33', thermal_dir / 'tsl-Al27inAl2O3.jeff33'), - (neutron_dir / '14-Si-28g.jeff33', thermal_dir / 'tsl-Silicon.jeff33'), - (neutron_dir / '20-Ca-40g.jeff33', thermal_dir / 'tsl-CainCaH2.jeff33'), - ] - - pwd = Path.cwd() - - destination = args.destination.resolve() - (destination / 'photon').mkdir(parents=True, exist_ok=True) - - with tempfile.TemporaryDirectory() as tmpdir: - # Save current working directory and temporarily change dir - if args.tmpdir: - os.chdir(tmpdir) - library = openmc.data.DataLibrary() - - # ========================================================================= - # DOWNLOAD FILES - if args.download: - for base, fname, checksum in files: - download(urljoin(base, fname), checksum) - - # ========================================================================= - # EXTRACT ARCHIVES - - if args.extract: - for _, f, _ in files: - if f.endswith('.zip'): - print(f'Extracting {f}...') - with zipfile.ZipFile(f) as zipf: - zipf.extractall() - elif f.endswith('.tar.gz') or f.endswith('.tgz'): - print(f'Extracting {f}...') - with tarfile.open(f) as tgz: - tgz.extractall() - - # ========================================================================= - # REPLACE C13 AND O17 WITH FILES FROM TENDL 2019 - - # The evaluations for C13 and O17 can't be processed by recent versions of - # NJOY 2016 because of a bug in MF=6 in the evaluations. Instead of trying - # to patch the evaluation, we just replace it with more recent evaluations - # from TENDL 2019 - - for url in tendl_files: - download(url, output_path=neutron_dir) - - (neutron_dir / 'n-C013.tendl').rename(neutron_dir / '6-C-13g.jeff33') - (neutron_dir / 'n-O017.tendl').rename(neutron_dir / '8-O-17g.jeff33') - - # ========================================================================= - # PROCESS INCIDENT NEUTRON AND THERMAL SCATTERING DATA IN PARALLEL - - with Pool() as pool: - neutron_paths = sorted(neutron_dir.glob('*.jeff33')) - results = [] - tracker = ProgressTracker( - len(neutron_paths) + len(thermal_paths), verb='Processed') - for p in neutron_paths: - func_args = (p, destination, args.libver, args.temperatures) - r = pool.apply_async( - process_neutron, func_args, - callback=tracker.callback(p.name), - error_callback=tracker.callback(p.name), - ) - results.append(r) - for p_neut, p_therm in thermal_paths: - func_args = (p_neut, p_therm, destination, args.libver) - r = pool.apply_async( - process_thermal, func_args, - callback=tracker.callback(p_therm.name), - error_callback=tracker.callback(p_therm.name), - ) - results.append(r) - for r in results: - r.wait() - - for p in sorted(destination.glob('*.h5'), key=sort_key): - library.register_file(p) - - # ========================================================================= - # INCIDENT PHOTON DATA - - for z in range(1, 101): - element = openmc.data.ATOMIC_SYMBOL[z] - print('Generating HDF5 file for Z={} ({})...'.format(z, element)) - - # Generate instance of IncidentPhoton - photo_file = photoat_dir / f'photoat-{z:03}_{element}_000.endf' - atom_file = atomic_dir / f'atom-{z:03}_{element}_000.endf' - data = openmc.data.IncidentPhoton.from_endf(photo_file, atom_file) - - # Write HDF5 file and register it - outfile = destination / 'photon' / f'{element}.h5' - data.export_to_hdf5(outfile, 'w', libver=args.libver) - library.register_file(outfile) - - library.export_to_xml(destination / 'cross_sections.xml') - - # Change back to original directory - if args.tmpdir: - os.chdir(pwd) - - -if __name__ == '__main__': - main() diff --git a/src/openmc_data/urls_jeff.py b/src/openmc_data/urls_jeff.py new file mode 100644 index 0000000..f0ab666 --- /dev/null +++ b/src/openmc_data/urls_jeff.py @@ -0,0 +1,151 @@ +"""ENDF inputs for JEFF generation, separate from the processed ACE registry. + +NEA record metadata and archive previews were checked on 2026-10-20. +The JEFF 4.0 pairs follow openmc-dev/data's generate_jeff.py (128 pairs). +In particular, ``tsl_O_D20.jeff`` really uses a zero in the archive name. +The additional temperature-split PuO2/UO2/Zy4 evaluations are not mapped by +that upstream generator and are deliberately not included here. +""" + + +THERMAL_PAIRS_33 = ( + ('1-H-1g.jeff33', 'tsl-HinCaH2.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinCH2.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinH2O.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinIce.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinMesitylene-PhaseII.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinOrthoH.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinParaH.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinToluene.jeff33'), + ('1-H-1g.jeff33', 'tsl-HinZrH.jeff33'), + ('1-H-2g.jeff33', 'tsl-DinD2O.jeff33'), + ('1-H-2g.jeff33', 'tsl-DinOrthoD.jeff33'), + ('1-H-2g.jeff33', 'tsl-DinParaD.jeff33'), + ('4-Be-9g.jeff33', 'tsl-Be.jeff33'), + ('6-C-0g.jeff33', 'tsl-Graphite.jeff33'), + ('8-O-16g.jeff33', 'tsl-O16inAl2O3.jeff33'), + ('8-O-16g.jeff33', 'tsl-OinD2O.jeff33'), + ('12-Mg-24g.jeff33', 'tsl-Mg.jeff33'), + ('13-Al-27g.jeff33', 'tsl-Al27inAl2O3.jeff33'), + ('14-Si-28g.jeff33', 'tsl-Silicon.jeff33'), + ('20-Ca-40g.jeff33', 'tsl-CainCaH2.jeff33'), +) + +# Grouped by incident evaluation; material suffixes retain the archive spelling. +_THERMAL_MATERIALS_40 = { + '1-H-001g': '''H_Benzene H_C5O2H8 H_C8H8 H_CaH2 H_CaO2H2 H_CH2 + H_CH4-liquid H_CH4-solid H_H2O H_HF H_Ice H_KOH H_LiH H_Mesitylene + H_MgH2 H_MgOH2 H_NaMgH3 H_NaOH H_ortho-H H_para-H H_ParaffinicOil + H_SrH2 H_Toluene H_UH3 H_YH2 H_ZrH H_ZrH15 H_ZrH2''', + '1-H-002g': 'D_7LiD D_D2O D_MgD2 D_ortho-D D_para-D', + '3-Li-007g': '7Li_7LiD Li_FLiBe Li_LiF Li_LiH', + '4-Be-009g': 'Be_Be Be_Be2C Be_BeF2 Be_BeO Be_FLiBe', + '6-C-012g': 'C_Be2C C_C5O2H8 C_C8H8 C_CF2 C_CH2 C_Diamond C_Graphite C_UC C_ZrC', + '7-N-014g': 'N_GaN N_UN', + '8-O-016g': '''O_Al2O3 O_BeO O_C5O2H8 O_CaO2H2 O_D20 O_Ge3Bi4O12 + O_KOH O_MgO O_MgOH2 O_NaOH O_Y3Al5O12''', + '9-F-019g': 'F_BeF2 F_CF2 F_FLiBe F_LiF F_MgF2', + '11-Na-023g': 'Na_Na Na_NaI Na_NaMgH3 Na_NaOH', + '12-Mg-024g': 'Mg_Mg Mg_MgD2 Mg_MgF2 Mg_MgH2 Mg_MgO Mg_MgOH2 Mg_NaMgH3', + '13-Al-027g': 'Al_Al Al_Al2O3 Al_Y3Al5O12', + '14-Si-028g': 'Si_Si', + '16-S-032g': 'S_ZnS', + '19-K-039g': 'K_K K_KOH', + '20-Ca-040g': 'Ca_Ca Ca_CaH2 Ca_CaO2H2', + '22-Ti-048g': 'Ti_Ti', + '23-V-051g': 'V_V', + '24-Cr-052g': 'Cr_Cr', + '26-Fe-056g': 'Fe_Fe-alpha Fe_Fe-gamma', + '28-Ni-058g': 'Ni_Ni', + '29-Cu-063g': 'Cu_Cu', + '30-Zn-064g': 'Zn_Zn Zn_ZnS', + '31-Ga-069g': 'Ga_GaN Ga_GaSe', + '32-Ge-074g': 'Ge_Ge Ge_Ge3Bi4O12 Ge_GeTe', + '34-Se-080g': 'Se_GaSe', + '38-Sr-088g': 'Sr_SrH2', + '39-Y-089g': 'Y_Y3Al5O12 Y_YH2', + '40-Zr-090g': 'Zr_Zr Zr_ZrC Zr_ZrH15 Zr_ZrH2', + '41-Nb-093g': 'Nb_Nb', + '42-Mo-098g': 'Mo_Mo', + '46-Pd-106g': 'Pd_Pd', + '47-Ag-107g': 'Ag_Ag', + '50-Sn-120g': 'Sn_Sn', + '52-Te-130g': 'Te_GeTe', + '53-I-127g': 'I_NaI', + '74-W-184g': 'W_W', + '78-Pt-195g': 'Pt_Pt', + '79-Au-197g': 'Au_Au', + '82-Pb-208g': 'Pb_Pb', + '83-Bi-209g': 'Bi_Bi Bi_Ge3Bi4O12', + '92-U-238g': 'U_U U_UC U_UN', +} +THERMAL_PAIRS_40 = tuple( + (f'n_{neutron}.jeff', f'tsl_{material}.jeff') + for neutron, materials in _THERMAL_MATERIALS_40.items() + for material in materials.split() +) + + +jeff_endf_details = { + '3.3': { + 'neutron': { + 'base_url': 'https://data.oecd-nea.org/api/records/bh7jn-rm903/files/', + 'compressed_files': ['JEFF33-n.tgz'], + 'url_suffix': '/content', + 'checksums': ['e540bbf95179257280c61acfa75c83de'], + 'pattern': '*.jeff33', + }, + 'thermal': { + 'base_url': 'https://data.oecd-nea.org/api/records/3nm84-46r21/files/', + 'compressed_files': ['JEFF33-tsl.tgz'], + 'url_suffix': '/content', + 'checksums': ['82a6df4cb802aa4a09b95309f7861c54'], + 'pairs': THERMAL_PAIRS_33, + }, + 'photon': { + 'base_url': 'https://www.nndc.bnl.gov/endf-b8.0/zips/', + 'compressed_files': ['ENDF-B-VIII.0_photoat.zip', 'ENDF-B-VIII.0_atomic_relax.zip'], + 'checksums': ['d49f5b54be278862e1ce742ccd94f5c0', 'e04d50098cb2a7e4fe404ec4071611cc'], + }, + }, + '4.0': { + 'neutron': { + 'base_url': 'https://data.oecd-nea.org/api/records/e9ajn-a3p20/files/', + 'compressed_files': ['JEFF40-Evaluations-Neutron-593.zip'], + 'url_suffix': '/content', + 'checksums': ['51d00ee7bf1491d428f9b30a9782e41d'], + 'pattern': '*.jeff', + }, + 'thermal': { + 'base_url': 'https://data.oecd-nea.org/api/records/scmva-nqh68/files/', + 'compressed_files': ['JEFF40-Evaluations-TSL.zip'], + 'url_suffix': '/content', + 'checksums': ['13d22cd59f83368885ab2d86300b470a'], + 'pairs': THERMAL_PAIRS_40, + }, + 'photon': { + 'base_url': 'https://www.nndc.bnl.gov/endf-releases/releases/B-VIII.1/', + 'compressed_files': ['photoat/photoat-version.VIII.1.tar.gz', + 'atomic_relax/atomic_relax-version.VIII.1.tar.gz'], + 'checksums': ['6d5f4830f6290d6c618803a8391ba0cf', '70e9ca0c481236499b7a3e0a490f4ef2'], + }, + }, +} + +# Individual IAEA archives avoid downloading the 3 GB official TENDL bulk tar. +# These replace defective JEFF 3.3 MF=6 evaluations, without modifying originals. +# IAEA does not publish checksums here; ZIP CRCs and the ENDF target are checked. +jeff33_replacements = { + '6-C-13g.jeff33': { + 'base_url': 'https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/', + 'compressed_files': ['n_6-C-13_0628.zip'], + 'checksums': [None], + 'target': (6, 13), + }, + '8-O-17g.jeff33': { + 'base_url': 'https://www-nds.iaea.org/public/download-endf/TENDL-2019/n/', + 'compressed_files': ['n_8-O-17_0828.zip'], + 'checksums': [None], + 'target': (8, 17), + }, +} \ No newline at end of file diff --git a/src/openmc_data/utils.py b/src/openmc_data/utils.py index 44eebd9..b458df2 100644 --- a/src/openmc_data/utils.py +++ b/src/openmc_data/utils.py @@ -215,6 +215,15 @@ def extract( Path(compressed_file).unlink(missing_ok=True) +def _file_md5(path): + """Hash large archives without loading them into memory.""" + checksum = hashlib.md5() + with path.open('rb') as stream: + for chunk in iter(lambda: stream.read(_BLOCK_SIZE), b''): + checksum.update(chunk) + return checksum.hexdigest() + + def download( url: str, checksum=None, @@ -268,8 +277,9 @@ def download( local_path = Path(output_path) / output_filename # Check if file already downloaded - if local_path.is_file(): - if local_path.stat().st_size == file_size: + if file_size is not None and local_path.is_file(): + if (local_path.stat().st_size == file_size + and (checksum is None or _file_md5(local_path) == checksum)): print(f'Skipping {local_path}, already downloaded') return local_path @@ -283,13 +293,19 @@ def download( break fh.write(chunk) downloaded += len(chunk) - status = '{:10} [{:3.2f}%]'.format( - downloaded, downloaded * 100. / file_size) + if file_size: + status = '{:10} [{:3.2f}%]'.format( + downloaded, downloaded * 100. / file_size) + else: + status = f'{downloaded:10} bytes' print(status + '\b'*len(status), end='', flush=True) print('') + if file_size is not None and downloaded != file_size: + raise OSError(f"Incomplete download of {url}: expected {file_size} bytes, received {downloaded}") + if checksum is not None: - downloadsum = hashlib.md5(open(local_path, 'rb').read()).hexdigest() + downloadsum = _file_md5(local_path) if downloadsum != checksum: raise OSError("MD5 checksum for {} does not match. If this is " "your first time receiving this message, please "