From fa85aecda2007da4ca839b6e3e0b689e67cac304 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 17 Jul 2026 16:02:56 +0100 Subject: [PATCH 01/37] DocIndex and FileIndex SaxParsers and use MapDB for parsed data --- .../SIARDDK1007ContentImportStrategy.java | 54 +---- .../content/SIARDDK1007DocIndexHandler.java | 56 +++++ .../SIARDDK128ContentImportStrategy.java | 56 +---- .../in/content/SIARDDK128DocIndexHandler.java | 56 +++++ .../content/SIARDDKContentImportStrategy.java | 95 ++++---- .../siard/in/content/SIARDDKDocIndexDoc.java | 80 +++++++ .../in/content/SIARDDKDocIndexHandler.java | 181 +++++++++++++++ .../SIARDDK1007ExtMetadataImportStrategy.java | 2 +- .../SIARDDK1007MetadataImportStrategy.java | 2 +- .../SIARDDK128ExtMetadataImportStrategy.java | 2 +- .../SIARDDK128MetadataImportStrategy.java | 2 +- .../SIARDDK1007ExtPathImportStrategy.java | 5 +- .../in/path/SIARDDK1007FileIndexHandler.java | 35 +++ .../path/SIARDDK1007PathImportStrategy.java | 21 +- .../path/SIARDDK128ExtPathImportStrategy.java | 5 +- .../in/path/SIARDDK128FileIndexHandler.java | 35 +++ .../in/path/SIARDDK128PathImportStrategy.java | 12 +- .../siard/in/path/SIARDDKFileIndexFile.java | 43 ++++ .../in/path/SIARDDKFileIndexHandler.java | 153 +++++++++++++ .../in/path/SIARDDKPathImportStrategy.java | 207 ++++++++---------- 20 files changed, 809 insertions(+), 293 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexFile.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java index 8f94967f8..4f2292090 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java @@ -32,7 +32,6 @@ import dk.sa.xmlns.diark._1_0.docindex.DocIndexType; import dk.sa.xmlns.diark._1_0.docindex.DocumentType; import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Unmarshaller; @@ -48,56 +47,9 @@ public SIARDDK1007ContentImportStrategy(FolderReadStrategyMD5Sum readStrategy, S super(readStrategy, pathStrategy, importAsSchema); } - DocIndexType loadVirtualTableContent() throws ModuleException, FileNotFoundException { - JAXBContext context; - try { - context = JAXBContext.newInstance(DocIndexType.class.getPackage().getName()); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error loading JAXBContext").withCause(e); - } - - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema xsdSchema = null; - InputStream xsdInputStream = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)); - - try { - xsdSchema = schemaFactory.newSchema(new StreamSource(xsdInputStream)); - } catch (SAXException e) { - throw new ModuleException() - .withMessage("Error reading metadata XSD file: " + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)) - .withCause(e); - } - InputStream inputStreamXml = null; - Unmarshaller unmarshaller; - try { - unmarshaller = context.createUnmarshaller(); - unmarshaller.setSchema(xsdSchema); - inputStreamXml = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX)); - Object result = unmarshaller.unmarshal(inputStreamXml); - DocIndexType docIndex; - if (result instanceof JAXBElement) { - docIndex = ((JAXBElement) result).getValue(); - } else if (result instanceof DocIndexType) { - docIndex = (DocIndexType) result; - } else { - throw new IllegalArgumentException("Unexpected object type: " + result.getClass().getName()); - } - return docIndex; - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error while Unmarshalling JAXB").withCause(e); - } finally { - try { - xsdInputStream.close(); - if (inputStreamXml != null) { - inputStreamXml.close(); - xsdInputStream.close(); - } - } catch (IOException e) { - logger.debug("Could not close xsdStream", e); - } - } + @Override + SIARDDKDocIndexHandler createDocIndexContentHandler(Schema xsdSchema) { + return new SIARDDK1007DocIndexHandler(pathStrategy, dbExportHandler); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java new file mode 100644 index 000000000..e50026fa7 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java @@ -0,0 +1,56 @@ +package com.databasepreservation.modules.siard.in.content; + +import com.databasepreservation.model.modules.DatabaseExportModule; +import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK1007DocIndexHandler extends SIARDDKDocIndexHandler { + + public SIARDDK1007DocIndexHandler(SIARDDKPathImportStrategy pathImportStrategy, + DatabaseExportModule databaseExportModule) { + super(pathImportStrategy, databaseExportModule); + } + + @Override + String getDocLocalName() { + return "doc"; + } + + @Override + String getDocIDLocalName() { + return "dID"; + } + + @Override + String getParentIDLocalName() { + return "pID"; + } + + @Override + String getMediaIDLocalName() { + return "mID"; + } + + @Override + String getContainerFolderLocalName() { + return "dCf"; + } + + @Override + String getOriginalFilenameLocalName() { + return "oFn"; + } + + @Override + String getArchivalFileTypeLocalName() { + return "aFt"; + } + + @Override + String getGmlXSDLocalName() { + return "gmlXsd"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java index fc376901c..85550112d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java @@ -24,15 +24,14 @@ import org.xml.sax.SAXException; import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.bindings.siard_dk_128.context.ContextDocumentationIndex; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocumentType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.context.ContextDocumentationIndex; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; import com.databasepreservation.modules.siard.in.read.FolderReadStrategyMD5Sum; import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Unmarshaller; @@ -54,56 +53,9 @@ public SIARDDK128ContentImportStrategy(FolderReadStrategyMD5Sum readStrategy, SI super(readStrategy, pathStrategy, importAsSchema); } - DocIndexType loadVirtualTableContent() throws ModuleException, FileNotFoundException { - JAXBContext context; - try { - context = JAXBContext.newInstance(DocIndexType.class.getPackage().getName()); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error loading JAXBContext").withCause(e); - } - - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema xsdSchema = null; - InputStream xsdInputStream = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)); - - try { - xsdSchema = schemaFactory.newSchema(new StreamSource(xsdInputStream)); - } catch (SAXException e) { - throw new ModuleException() - .withMessage("Error reading metadata XSD file: " + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)) - .withCause(e); - } - InputStream inputStreamXml = null; - Unmarshaller unmarshaller; - try { - unmarshaller = context.createUnmarshaller(); - unmarshaller.setSchema(xsdSchema); - inputStreamXml = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX)); - Object result = unmarshaller.unmarshal(inputStreamXml); - DocIndexType docIndex; - if (result instanceof JAXBElement) { - docIndex = ((JAXBElement) result).getValue(); - } else if (result instanceof DocIndexType) { - docIndex = (DocIndexType) result; - } else { - throw new IllegalArgumentException("Unexpected object type: " + result.getClass().getName()); - } - return docIndex; - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error while Unmarshalling JAXB").withCause(e); - } finally { - try { - xsdInputStream.close(); - if (inputStreamXml != null) { - inputStreamXml.close(); - xsdInputStream.close(); - } - } catch (IOException e) { - logger.debug("Could not close xsdStream", e); - } - } + @Override + SIARDDKDocIndexHandler createDocIndexContentHandler(Schema xsdSchema) { + return new SIARDDK128DocIndexHandler(pathStrategy, dbExportHandler); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java new file mode 100644 index 000000000..cfaae5d25 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java @@ -0,0 +1,56 @@ +package com.databasepreservation.modules.siard.in.content; + +import com.databasepreservation.model.modules.DatabaseExportModule; +import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK128DocIndexHandler extends SIARDDKDocIndexHandler { + + public SIARDDK128DocIndexHandler(SIARDDKPathImportStrategy pathImportStrategy, + DatabaseExportModule databaseExportModule) { + super(pathImportStrategy, databaseExportModule); + } + + @Override + String getDocLocalName() { + return "doc"; + } + + @Override + String getDocIDLocalName() { + return "dID"; + } + + @Override + String getParentIDLocalName() { + return "pID"; + } + + @Override + String getMediaIDLocalName() { + return "mID"; + } + + @Override + String getContainerFolderLocalName() { + return "dCf"; + } + + @Override + String getOriginalFilenameLocalName() { + return "oFn"; + } + + @Override + String getArchivalFileTypeLocalName() { + return "aFt"; + } + + @Override + String getGmlXSDLocalName() { + return "gmlXsd"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java index 011c33e46..3812372eb 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java @@ -8,8 +8,10 @@ package com.databasepreservation.modules.siard.in.content; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; @@ -299,70 +301,47 @@ void populateContextDocumentationTable(TableStructure table) throws ModuleExcept } void populateVirtualTable(TableStructure table) throws ModuleException, FileNotFoundException { - D docIndex = loadVirtualTableContent(); - currentTable = table; this.dbExportHandler.handleDataOpenTable(table.getId()); - int rowCounter = 0; - for (T doc : getDocuments(docIndex)) { - Row row = new Row(); - List lstCells = new ArrayList<>(); - row.setIndex(rowCounter); - - // document id - Cell dIDCell = new SimpleCell(SIARDDKConstants.DID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, - getDID(doc).toString()); - lstCells.add(dIDCell); - - // parent id - BigInteger pID = getPID(doc); - String pIDString = pID == null ? "" : pID.toString(); - - Cell pIDCell = new SimpleCell(SIARDDKConstants.PID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, - pIDString); - lstCells.add(pIDCell); - - try { - // document blob - String mainFolder = pathStrategy.getMainFolder().getPath().toString(); - String siardFolderName = mainFolder.substring(0, mainFolder.length() - 1) + getMID(doc); - Path siardFolderPath = Paths.get(siardFolderName); - Path docPath = siardFolderPath - .resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME, getDCf(doc), getDID(doc).toString())); - - if (!docPath.startsWith(siardFolderPath.resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME)))) { - throw new ModuleException().withMessage("Invalid path for folder: " + docPath); - } - - String digest = ""; - File docFolder = new File(docPath.toString()); - if (docFolder.exists() && docFolder.isDirectory()) { - File[] fileList = docFolder.listFiles(); - if (fileList != null && fileList.length == 1) { - docPath = docPath.resolve(Paths.get(fileList[0].getName())); - digest = DigestUtils.sha1Hex(Files.newInputStream(docPath)); - } - } - - Cell blobCell = new BinaryCell( - SIARDDKConstants.BLOB_EXTENSION + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, - new DummyInputStreamProvider(), docPath.toString(), Files.size(docPath), digest, - DigestUtils.getSha1Digest().toString()); - lstCells.add(blobCell); + currentTable = table; + loadVirtualTableContent(); + this.dbExportHandler.handleDataCloseTable(table.getId()); + } - // set and handle row - assert !lstCells.contains(null); - row.setCells(lstCells); - this.dbExportHandler.handleDataRow(row); + void loadVirtualTableContent() throws ModuleException, FileNotFoundException { + ValidatorHandler validatorHandler; + + try (InputStream xsdStream = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() + + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX))){ + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema xsdSchema = schemaFactory.newSchema(new StreamSource(xsdStream)); + SIARDDKDocIndexHandler docIndexHandler = createDocIndexContentHandler(xsdSchema); + validatorHandler = createDocIndexValidatorHandler(xsdSchema, docIndexHandler); + } catch (IOException | SAXException e) { + throw new ModuleException().withMessage("Error while preparing doc index parser").withCause(e); + } - rowCounter++; - } catch (ModuleException | IOException e) { - throw new ModuleException().withMessage("Error handling data row index:" + rowCounter).withCause(e); - } + try { + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setValidating(false); + saxParserFactory.setNamespaceAware(true); + SAXParser saxParser = saxParserFactory.newSAXParser(); + XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setContentHandler(validatorHandler); + xmlReader.parse(new InputSource(readStrategy.createInputStream(pathStrategy.getMainFolder(), + pathStrategy.getXmlFilePath(SIARDDKConstants.FILE_INDEX)))); + } catch (SAXException | ParserConfigurationException | IOException e) { + throw new ModuleException().withMessage("Error while parsing doc index").withCause(e); } - this.dbExportHandler.handleDataCloseTable(table.getId()); } - abstract D loadVirtualTableContent() throws ModuleException, FileNotFoundException; + ValidatorHandler createDocIndexValidatorHandler(Schema xsdSchema, SIARDDKDocIndexHandler docIndexHandler) { + ValidatorHandler validatorHandler = xsdSchema.newValidatorHandler(); + validatorHandler.setContentHandler(docIndexHandler); + return validatorHandler; + } + + abstract SIARDDKDocIndexHandler createDocIndexContentHandler(Schema xsdSchema); + abstract F loadContextDocTableContent() throws ModuleException, FileNotFoundException; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java new file mode 100644 index 000000000..7deed4963 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java @@ -0,0 +1,80 @@ +package com.databasepreservation.modules.siard.in.content; + +import java.io.Serializable; +import java.math.BigInteger; + +/** + * + * @author Alexandre Flores + */ + +public class SIARDDKDocIndexDoc implements Serializable { + BigInteger documentID; + BigInteger parentID; + BigInteger mediaID; + String documentCollectionFolder; + String originalFilename; + String archivalFileType; + String gmlXSD; + + public SIARDDKDocIndexDoc() { + + } + + public BigInteger getParentID() { + return parentID; + } + + public void setParentID(BigInteger parentID) { + this.parentID = parentID; + } + + public BigInteger getDocumentID() { + return documentID; + } + + public void setDocID(BigInteger documentID) { + this.documentID = documentID; + } + + public BigInteger getMediaID() { + return mediaID; + } + + public void setMediaID(BigInteger mediaID) { + this.mediaID = mediaID; + } + + public String getDocumentCollectionFolder() { + return documentCollectionFolder; + } + + public void setContainerFolder(String documentCollectionFolder) { + this.documentCollectionFolder = documentCollectionFolder; + } + + public String getOriginalFilename() { + return originalFilename; + } + + public void setOriginalFilename(String originalFilename) { + this.originalFilename = originalFilename; + } + + public String getArchivalFileType() { + return archivalFileType; + } + + public void setArchivalFileType(String archivalFileType) { + this.archivalFileType = archivalFileType; + } + + public String getGmlXSD() { + return gmlXSD; + } + + public void setGmlXSD(String gmlXSD) { + this.gmlXSD = gmlXSD; + } + +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java new file mode 100644 index 000000000..3355a33fd --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java @@ -0,0 +1,181 @@ +package com.databasepreservation.modules.siard.in.content; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; +import org.apache.commons.codec.digest.DigestUtils; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import com.databasepreservation.common.io.providers.DummyInputStreamProvider; +import com.databasepreservation.model.data.BinaryCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.model.data.Row; +import com.databasepreservation.model.data.SimpleCell; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.modules.DatabaseExportModule; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + +/** + * + * @author Alexandre Flores + */ +public abstract class SIARDDKDocIndexHandler extends DefaultHandler { + protected SIARDDKDocIndexDoc currentDoc; + protected StringBuilder currentElementCharacters; + + private SIARDDKPathImportStrategy pathImportStrategy; + private DatabaseExportModule databaseExportModule; + private int rowCounter; + + public SIARDDKDocIndexHandler(SIARDDKPathImportStrategy pathImportStrategy, + DatabaseExportModule databaseExportModule) { + this.pathImportStrategy = pathImportStrategy; + this.databaseExportModule = databaseExportModule; + rowCounter = 0; + } + + @Override + public void startDocument() throws SAXException { + // no op + } + + @Override + public void endDocument() throws SAXException { + // no op + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + currentElementCharacters = new StringBuilder(); + + if (localName.equals(getDocLocalName())) { + startElementDoc(uri, localName, qName, attributes); + } + } + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + if (localName.equals(getDocLocalName())) { + endElementDoc(); + } + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + currentElementCharacters.append(ch, start, length); + } + + protected void startElementDoc(String uri, String localName, String qName, Attributes attributes) { + this.currentDoc = new SIARDDKDocIndexDoc(); + } + + protected void endElementDoc() throws SAXException { + Row row = new Row(); + List lstCells = new ArrayList<>(); + row.setIndex(rowCounter); + + // document id + Cell dIDCell = new SimpleCell(SIARDDKConstants.DID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, + currentDoc.getDocumentID().toString()); + lstCells.add(dIDCell); + + // parent id + BigInteger pID = currentDoc.getParentID(); + String pIDString = pID == null ? "" : pID.toString(); + + Cell pIDCell = new SimpleCell(SIARDDKConstants.PID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, + pIDString); + lstCells.add(pIDCell); + + try { + // document blob + String mainFolder = pathImportStrategy.getMainFolder().getPath().toString(); + String siardFolderName = mainFolder.substring(0, mainFolder.length() - 1) + currentDoc.getMediaID(); + Path siardFolderPath = Paths.get(siardFolderName); + Path docPath = siardFolderPath.resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME, + currentDoc.getDocumentCollectionFolder(), currentDoc.getDocumentID().toString())); + + if (!docPath.startsWith(siardFolderPath.resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME)))) { + throw new ModuleException().withMessage("Invalid path for folder: " + docPath); + } + + String digest = ""; + File docFolder = new File(docPath.toString()); + if (docFolder.exists() && docFolder.isDirectory()) { + File[] fileList = docFolder.listFiles(); + if (fileList != null && fileList.length == 1) { + docPath = docPath.resolve(Paths.get(fileList[0].getName())); + digest = DigestUtils.sha1Hex(Files.newInputStream(docPath)); + } + } + + Cell blobCell = new BinaryCell( + SIARDDKConstants.BLOB_EXTENSION + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, + new DummyInputStreamProvider(), docPath.toString(), Files.size(docPath), digest, + DigestUtils.getSha1Digest().toString()); + lstCells.add(blobCell); + + // set and handle row + assert !lstCells.contains(null); + row.setCells(lstCells); + this.databaseExportModule.handleDataRow(row); + + rowCounter++; + } catch (ModuleException | IOException e) { + throw new SAXException("Error handling data row index:" + rowCounter, e); + } + } + + protected void endElementDocID() { + this.currentDoc.setDocID(new BigInteger(this.currentElementCharacters.toString())); + } + + protected void endElementParentID() { + this.currentDoc.setParentID(new BigInteger(this.currentElementCharacters.toString())); + } + + protected void endElementMediaID() { + this.currentDoc.setMediaID(new BigInteger(this.currentElementCharacters.toString())); + } + + protected void endElementContainerFolder() { + this.currentDoc.setContainerFolder(this.currentElementCharacters.toString()); + } + + protected void endElementOriginalFilename() { + this.currentDoc.setOriginalFilename(this.currentElementCharacters.toString()); + } + + protected void endElementArchivalFileType() { + this.currentDoc.setArchivalFileType(this.currentElementCharacters.toString()); + } + + protected void endElementGmlXSD() { + this.currentDoc.setGmlXSD(this.currentElementCharacters.toString()); + } + + abstract String getDocLocalName(); + + abstract String getDocIDLocalName(); + + abstract String getParentIDLocalName(); + + abstract String getMediaIDLocalName(); + + abstract String getContainerFolderLocalName(); + + abstract String getOriginalFilenameLocalName(); + + abstract String getArchivalFileTypeLocalName(); + + abstract String getGmlXSDLocalName(); +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java index 8af36d970..6e1e660bb 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java @@ -143,7 +143,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java index e1d7289a9..682c6e136 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java @@ -142,7 +142,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java index 16ee2857d..10dd689da 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java @@ -142,7 +142,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java index b50faefab..c02344dd1 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java @@ -142,7 +142,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java index acd8386f6..f6d341df8 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java @@ -32,8 +32,9 @@ public SIARDDK1007ExtPathImportStrategy(SIARDArchiveContainer mainFolder, ReadSt } @Override - byte[] getMd5(F fileInfo) { - return fileInfo.getMd5(); + SIARDDKFileIndexHandler createFileIndexHandler() { + return new SIARDDK1007FileIndexHandler(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, + xmlFilePathLookupByFolderName); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java new file mode 100644 index 000000000..e46064ec3 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java @@ -0,0 +1,35 @@ +package com.databasepreservation.modules.siard.in.path; + +import java.util.Map; + +/** + * @author Alexandre Flores + */ +public class SIARDDK1007FileIndexHandler extends SIARDDKFileIndexHandler { + + public SIARDDK1007FileIndexHandler(Map archiveFolderLookupByFolderName, + Map xsdFilePathLookupByFolderName, + Map xmlFilePathLookupByFolderName) { + super(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, xmlFilePathLookupByFolderName); + } + + @Override + String getFileLocalName() { + return "f"; + } + + @Override + String getFileNameLocalName() { + return "fiN"; + } + + @Override + String getFolderNameLocalName() { + return "foN"; + } + + @Override + String getMD5LocalName() { + return "md5"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java index 20969bcd7..7c493c5e1 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java @@ -8,14 +8,15 @@ package com.databasepreservation.modules.siard.in.path; import java.util.List; -import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType; -import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType.F; -import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; -import com.databasepreservation.modules.siard.in.read.ReadStrategy; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType; +import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType.F; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.in.read.ReadStrategy; /** * @author António Lindo @@ -23,13 +24,17 @@ public class SIARDDK1007PathImportStrategy extends SIARDDKPathImportStrategy { protected final Logger logger = LoggerFactory.getLogger(ContentPathImportStrategy.class); - public SIARDDK1007PathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy readStrategy, MetadataPathStrategy metadataPathStrategy, String importAsSchema, FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy) { - super(mainFolder, readStrategy, metadataPathStrategy, importAsSchema, fileIndexXsdInputStreamStrategy, FileIndexType.class); + public SIARDDK1007PathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy readStrategy, + MetadataPathStrategy metadataPathStrategy, String importAsSchema, + FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy) { + super(mainFolder, readStrategy, metadataPathStrategy, importAsSchema, fileIndexXsdInputStreamStrategy, + FileIndexType.class); } @Override - byte[] getMd5(F fileInfo) { - return fileInfo.getMd5(); + SIARDDKFileIndexHandler createFileIndexHandler() { + return new SIARDDK1007FileIndexHandler(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, + xmlFilePathLookupByFolderName); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java index 7eab0d96b..1c5f92850 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java @@ -28,8 +28,9 @@ public SIARDDK128ExtPathImportStrategy(SIARDArchiveContainer mainFolder, ReadStr } @Override - byte[] getMd5(F fileInfo) { - return fileInfo.getMd5(); + SIARDDKFileIndexHandler createFileIndexHandler() { + return new SIARDDK128FileIndexHandler(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, + xmlFilePathLookupByFolderName); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java new file mode 100644 index 000000000..64622b23e --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java @@ -0,0 +1,35 @@ +package com.databasepreservation.modules.siard.in.path; + +import java.util.Map; + +/** + * @author Alexandre Flores + */ +public class SIARDDK128FileIndexHandler extends SIARDDKFileIndexHandler { + + public SIARDDK128FileIndexHandler(Map archiveFolderLookupByFolderName, + Map xsdFilePathLookupByFolderName, + Map xmlFilePathLookupByFolderName) { + super(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, xmlFilePathLookupByFolderName); + } + + @Override + String getFileLocalName() { + return "f"; + } + + @Override + String getFileNameLocalName() { + return "fiN"; + } + + @Override + String getFolderNameLocalName() { + return "foN"; + } + + @Override + String getMD5LocalName() { + return "md5"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java index 7f768485c..0aeeb28ba 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java @@ -7,14 +7,14 @@ */ package com.databasepreservation.modules.siard.in.path; +import java.util.List; + import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType.F; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; import com.databasepreservation.modules.siard.in.read.ReadStrategy; -import java.util.List; - /** * @author António Lindo */ @@ -23,12 +23,14 @@ public class SIARDDK128PathImportStrategy extends SIARDDKPathImportStrategy + */ + +public class SIARDDKFileIndexFile implements Serializable { + String folderName; + String fileName; + byte[] md5; + + public SIARDDKFileIndexFile() { + + } + + public String getFolderName() { + return folderName; + } + + public void setFolderName(String folderName) { + this.folderName = folderName; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public byte[] getMd5() { + return md5; + } + + public void setMd5(byte[] md5) { + this.md5 = md5; + } + +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java new file mode 100644 index 000000000..ef82fe522 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java @@ -0,0 +1,153 @@ +package com.databasepreservation.modules.siard.in.path; + +import java.nio.file.FileSystems; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + +/** + * + * @author Alexandre Flores + */ +public abstract class SIARDDKFileIndexHandler extends DefaultHandler { + + protected final Map archiveFolderLookupByFolderName; + protected final Map xmlFilePathLookupByFolderName; + protected final Map xsdFilePathLookupByFolderName; + protected byte[] tableIndexExpectedMD5Sum; + protected byte[] archiveIndexExpectedMD5Sum; + + private final Pattern patternTableFolder = Pattern + .compile("(AVID\\.[A-ZÆØÅ]{2,4}\\.[1-9][0-9]*\\.[1-9][0-9]*)\\\\Tables\\\\(table[0-9]*)"); + private final Pattern patternIndicesFolder = Pattern + .compile("AVID\\.[A-ZÆØÅ]{2,4}\\.[1-9][0-9]*\\.[1-9][0-9]*\\\\Indices"); + + protected SIARDDKFileIndexFile currentFile; + protected StringBuilder currentElementCharacters; + + public SIARDDKFileIndexHandler(Map archiveFolderLookupByFolderName, + Map xsdFilePathLookupByFolderName, + Map xmlFilePathLookupByFolderName) { + this.archiveFolderLookupByFolderName = archiveFolderLookupByFolderName; + this.xsdFilePathLookupByFolderName = xsdFilePathLookupByFolderName; + this.xmlFilePathLookupByFolderName = xmlFilePathLookupByFolderName; + } + + @Override + public void startDocument() throws SAXException { + // no op + } + + @Override + public void endDocument() throws SAXException { + // no op + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + currentElementCharacters = new StringBuilder(); + + if (localName.equals(getFileLocalName())) { + startElementFile(uri, localName, qName, attributes); + } + } + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + if (localName.equals(getFileLocalName())) { + endElementFile(); + } else if (localName.equals(getFileNameLocalName())) { + endElementFileName(); + } else if (localName.equals(getFolderNameLocalName())) { + endElementFolderName(); + } else if (localName.equals(getMD5LocalName())) { + endElementMD5(); + } + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + currentElementCharacters.append(ch, start, length); + } + + protected void startElementFile(String uri, String localName, String qName, Attributes attributes) { + this.currentFile = new SIARDDKFileIndexFile(); + } + + private void endElementFile() throws SAXException { + Matcher matcherTableFolder = patternTableFolder.matcher(currentFile.getFolderName()); + if (matcherTableFolder.matches()) { + String folderName = matcherTableFolder.group(2); + String archivePath = FileSystems.getDefault().getPath(matcherTableFolder.group(1)).toString(); + archiveFolderLookupByFolderName.put(folderName, archivePath); + if (currentFile.getFileName().toLowerCase().endsWith(SIARDDKConstants.XML_EXTENSION)) { + if (xmlFilePathLookupByFolderName.containsKey(folderName)) { + throw new SAXException("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX + + " for table files. Multiple entries for the xml file for folder [" + folderName + "]."); + } + xmlFilePathLookupByFolderName.put(folderName, currentFile); + } else { + if (currentFile.getFileName().toLowerCase().endsWith(SIARDDKConstants.XSD_EXTENSION)) { + if (xsdFilePathLookupByFolderName.containsKey(folderName)) { + throw new SAXException("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX + + " for table files. Multiple entries for the xsd file for folder [" + folderName + "]."); + } + xsdFilePathLookupByFolderName.put(folderName, currentFile); + } + } + } else { + Matcher mIndicesFldr = patternIndicesFolder.matcher(currentFile.getFolderName()); + if (mIndicesFldr.matches()) { + // please notice, that this is a rudimentary implementation, only + // considering the files relevant for the SIARDDK import module. + if (currentFile.getFileName().equals(SIARDDKConstants.TABLE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { + tableIndexExpectedMD5Sum = currentFile.getMd5(); + } else if (currentFile.getFileName() + .equals(SIARDDKConstants.ARCHIVE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { + archiveIndexExpectedMD5Sum = currentFile.getMd5(); + } + + } + } + } + + protected void endElementFileName() throws SAXException { + this.currentFile.setFileName(this.currentElementCharacters.toString()); + } + + protected void endElementFolderName() throws SAXException { + this.currentFile.setFolderName(this.currentElementCharacters.toString()); + } + + protected void endElementMD5() throws SAXException { + try { + this.currentFile.setMd5(Hex.decodeHex(this.currentElementCharacters.toString())); + } catch (DecoderException e) { + throw new SAXException("Unable to decode MD5 hex string", e); + } + } + + abstract String getFileLocalName(); + + abstract String getFileNameLocalName(); + + abstract String getFolderNameLocalName(); + + abstract String getMD5LocalName(); + + public byte[] getArchiveIndexExpectedMD5Sum() { + return archiveIndexExpectedMD5Sum; + } + + public byte[] getTableIndexExpectedMD5Sum() { + return tableIndexExpectedMD5Sum; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java index dc863335a..2d25efcb8 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java @@ -7,35 +7,44 @@ */ package com.databasepreservation.modules.siard.in.path; -import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; -import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; -import com.databasepreservation.modules.siard.in.read.ReadStrategy; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBElement; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Unmarshaller; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.XMLConstants; -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; import java.io.IOException; import java.io.InputStream; import java.nio.file.FileSystems; import java.nio.file.Path; +import java.nio.file.Paths; import java.security.InvalidParameterException; -import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.regex.Matcher; +import java.util.UUID; import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.ValidatorHandler; + +import org.mapdb.DB; +import org.mapdb.DBMaker; +import org.mapdb.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; + +import com.databasepreservation.Constants; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; +import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.in.read.ReadStrategy; +import com.databasepreservation.utils.ConfigUtils; + /** * @author Thomas Kristensen * @@ -45,26 +54,26 @@ * to retrieve md5sums.(The impl. of retrieval of md5sums for the meta * data files are only implemented to the extend that it is needed. ) */ -public abstract class SIARDDKPathImportStrategy implements ContentPathImportStrategy, MetadataPathStrategy { +public abstract class SIARDDKPathImportStrategy extends DefaultHandler + implements ContentPathImportStrategy, MetadataPathStrategy { protected final Logger logger = LoggerFactory.getLogger(ContentPathImportStrategy.class); protected final String importAsSchema; protected final SIARDArchiveContainer mainFolder; protected final ReadStrategy readStrategy; protected final MetadataPathStrategy metadataPathStrategy; - protected final Map xmlFilePathLookupByFolderName = new HashMap(); - protected final Map xsdFilePathLookupByFolderName = new HashMap(); - protected final Map folderNameLookupByTableId = new HashMap(); - protected final Map archiveFolderLookupByFolderName = new HashMap(); - - private FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy; - + protected final DB mapDB; + protected final Map xmlFilePathLookupByFolderName; + protected final Map xsdFilePathLookupByFolderName; + protected final Map folderNameLookupByTableId; + protected final Map archiveFolderLookupByFolderName; protected final Pattern folderSperatorPattern = Pattern.compile("[\\\\\\/]"); + private final Class fileIndexTypeClass; // protected byte[] fileIndexExpectedMD5Sum; --For some reason, no md5sum is // required for fileIndex.xml in the standard - protected byte[] tabelIndexExpectedMD5Sum; + protected byte[] tableIndexExpectedMD5Sum; protected byte[] archiveIndexExpectedMD5Sum; protected boolean fileIndexIsParsed; - private final Class fileIndexTypeClass; + private FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy; public SIARDDKPathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy readStrategy, MetadataPathStrategy metadataPathStrategy, String importAsSchema, @@ -76,102 +85,78 @@ public SIARDDKPathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy this.importAsSchema = importAsSchema; this.fileIndexXsdInputStreamStrategy = fileIndexXsdInputStreamStrategy; this.fileIndexTypeClass = fileIndexTypeClass; + + // Lookups maps + this.mapDB = setupMapDB(); + this.xmlFilePathLookupByFolderName = this.mapDB + .hashMap("xmlFilePathLookupByFolderName", Serializer.STRING, Serializer.JAVA).createOrOpen(); + this.xsdFilePathLookupByFolderName = this.mapDB + .hashMap("xsdFilePathLookupByFolderName", Serializer.STRING, Serializer.JAVA).createOrOpen(); + this.folderNameLookupByTableId = this.mapDB + .hashMap("xsdFilePathLookupByFolderName", Serializer.STRING, Serializer.STRING).createOrOpen(); + this.archiveFolderLookupByFolderName = this.mapDB + .hashMap("archiveFolderLookupByFolderName", Serializer.STRING, Serializer.JAVA).createOrOpen(); } public void parseFileIndexMetadata() throws ModuleException { if (!fileIndexIsParsed) { - JAXBContext context; - try { - context = JAXBContext.newInstance(fileIndexTypeClass.getPackage().getName()); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error loading JAXBContext").withCause(e); - } - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema xsdSchema = null; + Schema xsdSchema; InputStream xsdStream = fileIndexXsdInputStreamStrategy.getInputStream(this); + ValidatorHandler validatorHandler; + SIARDDKFileIndexHandler fileIndexHandler; try { xsdSchema = schemaFactory.newSchema(new StreamSource(xsdStream)); + validatorHandler = xsdSchema.newValidatorHandler(); + fileIndexHandler = createFileIndexHandler(); + validatorHandler.setContentHandler(fileIndexHandler); } catch (SAXException e) { throw new ModuleException() .withMessage( "Error reading metadata XSD file: " + metadataPathStrategy.getXsdFilePath(SIARDDKConstants.FILE_INDEX)) .withCause(e); } - InputStream reader = null; - D xmlFileIndex; - Unmarshaller unmarshaller; + try { - unmarshaller = context.createUnmarshaller(); - unmarshaller.setSchema(xsdSchema); - reader = readStrategy.createInputStream(mainFolder, - metadataPathStrategy.getXmlFilePath(SIARDDKConstants.FILE_INDEX)); - @SuppressWarnings("unchecked") - JAXBElement jaxbElement = (JAXBElement) unmarshaller.unmarshal(reader); - xmlFileIndex = jaxbElement.getValue(); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error while Unmarshalling JAXB").withCause(e); + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setValidating(false); + saxParserFactory.setNamespaceAware(true); + SAXParser saxParser = saxParserFactory.newSAXParser(); + XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setContentHandler(validatorHandler); + xmlReader.parse(new InputSource(readStrategy.createInputStream(mainFolder, + metadataPathStrategy.getXmlFilePath(SIARDDKConstants.FILE_INDEX)))); + } catch (SAXException | ParserConfigurationException | IOException e) { + throw new ModuleException().withMessage("Error while parsing file index").withCause(e); } finally { try { xsdStream.close(); - if (reader != null) { - reader.close(); - } } catch (IOException e) { logger.debug("Could not close xsdStream", e); } } - Pattern patternTableFolder = Pattern - .compile("(AVID\\.[A-ZÆØÅ]{2,4}\\.[0-9]*\\.[0-9]*)\\\\Tables\\\\(table[0-9]*)"); - - Pattern patternIndicesFolder = Pattern.compile("AVID\\.[A-ZÆØÅ]{2,4}\\.[0-9]*\\.1\\\\Indices"); - - for (T fileInfo : getF(xmlFileIndex)) { - Matcher mTblFldr = patternTableFolder.matcher(getFoN(fileInfo)); - if (mTblFldr.matches()) { - String folderName = mTblFldr.group(2); - Path archivePath = FileSystems.getDefault().getPath(mTblFldr.group(1)); - archiveFolderLookupByFolderName.put(folderName, archivePath); - if (getFiN(fileInfo).toLowerCase().endsWith(SIARDDKConstants.XML_EXTENSION)) { - if (xmlFilePathLookupByFolderName.containsKey(folderName)) { - throw new ModuleException().withMessage("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX - + " for table files. Multiple entries for the xml file for folder [" + folderName + "]."); - } - xmlFilePathLookupByFolderName.put(folderName, fileInfo); - } else { - if (getFiN(fileInfo).toLowerCase().endsWith(SIARDDKConstants.XSD_EXTENSION)) { - if (xsdFilePathLookupByFolderName.containsKey(folderName)) { - throw new ModuleException().withMessage("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX - + " for table files. Multiple entries for the xsd file for folder [" + folderName + "]."); - } - xsdFilePathLookupByFolderName.put(folderName, fileInfo); - } - } - } else { - Matcher mIndicesFldr = patternIndicesFolder.matcher(getFoN(fileInfo)); - if (mIndicesFldr.matches()) { - // please notice, that this is a rudimentary implementation, only - // considering the files relevant for the SIARDDK import module. - if (getFiN(fileInfo).equals(SIARDDKConstants.TABLE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { - tabelIndexExpectedMD5Sum = getMd5(fileInfo); - } else if (getFiN(fileInfo).equals(SIARDDKConstants.ARCHIVE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { - archiveIndexExpectedMD5Sum = getMd5(fileInfo); - } - /* - * else { if (fileInfo.getFiN().equals(SIARDDKConstants.FILE_INDEX + "." + - * SIARDDKConstants.XML_EXTENSION)) { fileIndexExpectedMD5Sum = - * fileInfo.getMd5(); } - */ - - } - } - } + tableIndexExpectedMD5Sum = fileIndexHandler.getTableIndexExpectedMD5Sum(); + archiveIndexExpectedMD5Sum = fileIndexHandler.getArchiveIndexExpectedMD5Sum(); + fileIndexIsParsed = true; } } + private DB setupMapDB() { + Path fileDBPath; + String fileDirectoryLocation = ConfigUtils.getProperty(Constants.PROPERTY_UNSET, "dbptk.memory.dir"); + if (fileDirectoryLocation.equals(Constants.PROPERTY_UNSET)) { + fileDBPath = Paths.get(ConfigUtils.getMapDBHomeDirectory().normalize().toAbsolutePath().toString(), + UUID.randomUUID().toString()); + } else { + fileDBPath = Paths.get(fileDirectoryLocation, UUID.randomUUID().toString()); + } + return DBMaker.fileDB(fileDBPath.toFile()).fileDeleteAfterClose().fileMmapEnable().fileMmapEnableIfSupported() + .fileMmapPreclearDisable().closeOnJvmShutdown().make(); + } + @Override public String getLobPath(String basePath, String schemaName, String tableId, String columnId, String lobFileName) { throw new UnsupportedOperationException("Invoking getLobPath(...) is not relevant for SIARDDK."); @@ -225,7 +210,7 @@ public String getTableXMLFilePath(String schemaName, String tableId) throws Modu } public byte[] getTableXMLFileMD5(String schemaName, String tableId) throws ModuleException { - return getMd5(getTableXMLFileInfo(schemaName, tableId)); + return getTableXMLFileInfo(schemaName, tableId).getMd5(); } public byte[] getArchiveIndexExpectedMD5Sum() throws ModuleException { @@ -245,25 +230,25 @@ protected void canLookupXSDFilePath(String folderName) throws ModuleException { } } - protected T getTableXMLFileInfo(String schemaName, String tableId) throws ModuleException { + protected SIARDDKFileIndexFile getTableXMLFileInfo(String schemaName, String tableId) throws ModuleException { canLookupTable(schemaName, tableId); String folderName = folderNameLookupByTableId.get(tableId); canLookupXMLFilePath(folderName); return xmlFilePathLookupByFolderName.get(folderName); } - protected T getTableXSDFileInfo(String schemaName, String tableId) throws ModuleException { + protected SIARDDKFileIndexFile getTableXSDFileInfo(String schemaName, String tableId) throws ModuleException { canLookupTable(schemaName, tableId); String folderName = folderNameLookupByTableId.get(tableId); canLookupXSDFilePath(folderName); return xsdFilePathLookupByFolderName.get(folderName); } - protected String buildPathSansArchiveFolderName(T fileInfo) { + protected String buildPathSansArchiveFolderName(SIARDDKFileIndexFile fileInfo) { Path pathFolderSperatorNeutral = FileSystems.getDefault().getPath("", - folderSperatorPattern.split(getFoN(fileInfo))); + folderSperatorPattern.split(fileInfo.getFolderName())); pathFolderSperatorNeutral = pathFolderSperatorNeutral.subpath(1, pathFolderSperatorNeutral.getNameCount()); - Path pathFolderSperatorNeutralWithFile = pathFolderSperatorNeutral.resolve(getFiN(fileInfo)); + Path pathFolderSperatorNeutralWithFile = pathFolderSperatorNeutral.resolve(fileInfo.getFileName()); return pathFolderSperatorNeutralWithFile.toString(); } @@ -273,14 +258,14 @@ public String getTableXSDFilePath(String schemaName, String tableId) throws Modu } public byte[] getTableXSDFileMD5(String schemaName, String tableId) throws ModuleException { - return getMd5(getTableXSDFileInfo(schemaName, tableId)); + return getTableXSDFileInfo(schemaName, tableId).getMd5(); } public Path getArchiveFolderPath(String schemaName, String tableId) throws ModuleException { canLookupTable(schemaName, tableId); String folderName = folderNameLookupByTableId.get(tableId); assert (archiveFolderLookupByFolderName.containsKey(folderName)); - return archiveFolderLookupByFolderName.get(folderName); + return Path.of(archiveFolderLookupByFolderName.get(folderName)); } @Override @@ -307,13 +292,13 @@ public String getXsdResourcePath(String filename) throws InvalidParameterExcepti * fileIndexExpectedMD5Sum; } */ - public byte[] getTabelIndexExpectedMD5Sum() throws ModuleException { - if (tabelIndexExpectedMD5Sum == null && fileIndexIsParsed) { + public byte[] getTableIndexExpectedMD5Sum() throws ModuleException { + if (tableIndexExpectedMD5Sum == null && fileIndexIsParsed) { throw new ModuleException() .withMessage("Parsing of " + SIARDDKConstants.FILE_INDEX + "." + SIARDDKConstants.XML_EXTENSION + " did not provide a md5sum for " + SIARDDKConstants.TABLE_INDEX + "." + SIARDDKConstants.XML_EXTENSION); } - return tabelIndexExpectedMD5Sum; + return tableIndexExpectedMD5Sum; } /** @@ -330,7 +315,7 @@ public SIARDArchiveContainer getMainFolder() { return mainFolder; } - abstract byte[] getMd5(T fileInfo); + abstract SIARDDKFileIndexHandler createFileIndexHandler(); abstract List getF(D fileIndex); From 5fae86f1f547a8f1da2f49d6fab5e27f13ae97e2 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Thu, 3 Sep 2026 11:38:34 +0100 Subject: [PATCH 02/37] Fix fatal issues with new doc index parsing --- .../integration/siard/SiardDKTestWrapper.java | 8 +++++++ .../content/SIARDDKContentImportStrategy.java | 2 +- .../in/content/SIARDDKDocIndexHandler.java | 21 +++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java index eb2a56cc7..b2d8be941 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java +++ b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java @@ -11,6 +11,9 @@ import java.util.ArrayList; import java.util.Iterator; +import com.databasepreservation.Main; +import com.databasepreservation.utils.ConfigUtils; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -30,6 +33,11 @@ @Test(groups = {"siarddk-roundtrip"}) public class SiardDKTestWrapper { + @BeforeClass + public void init() { + ConfigUtils.initialize(); + } + @DataProvider public Iterator siardVersionsProvider() { ArrayList tests = new ArrayList(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java index 3812372eb..eefe3da8b 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java @@ -328,7 +328,7 @@ void loadVirtualTableContent() throws ModuleException, FileNotFoundException { XMLReader xmlReader = saxParser.getXMLReader(); xmlReader.setContentHandler(validatorHandler); xmlReader.parse(new InputSource(readStrategy.createInputStream(pathStrategy.getMainFolder(), - pathStrategy.getXmlFilePath(SIARDDKConstants.FILE_INDEX)))); + pathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX)))); } catch (SAXException | ParserConfigurationException | IOException e) { throw new ModuleException().withMessage("Error while parsing doc index").withCause(e); } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java index 3355a33fd..3c9028805 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java @@ -67,6 +67,27 @@ public void endElement(String uri, String localName, String qName) throws SAXExc if (localName.equals(getDocLocalName())) { endElementDoc(); } + else if (localName.equals(getDocIDLocalName())) { + endElementDocID(); + } + else if (localName.equals(getContainerFolderLocalName())) { + endElementContainerFolder(); + } + else if (localName.equals(getGmlXSDLocalName())) { + endElementGmlXSD(); + } + else if (localName.equals(getMediaIDLocalName())) { + endElementMediaID(); + } + else if (localName.equals(getArchivalFileTypeLocalName())) { + endElementArchivalFileType(); + } + else if (localName.equals(getParentIDLocalName())) { + endElementParentID(); + } + else if (localName.equals(getOriginalFilenameLocalName())) { + endElementOriginalFilename(); + } } @Override From 3d33d358b7faa6dd0ef81738baaefc28fb7f6d2d Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 8 May 2026 16:16:05 +0100 Subject: [PATCH 03/37] WIP - SIARDDK export and import support --- .../modules/postgresql/PostgreSQLHelper.java | 2 +- .../SIARDDK1007FileIndexFileStrategy.java | 7 +++ .../SIARDDK128FileIndexFileStrategy.java | 7 +++ .../SIARDDKFileIndexFileStrategy.java | 7 ++- .../SIARDDKMetadataExportStrategy.java | 16 +++--- .../siard/out/metadata/SIARDMarshaller.java | 17 ++++++ .../out/metadata/StandardSIARDMarshaller.java | 54 +++++++++++++++++-- .../SIARDDK1007DatabaseExportModule.java | 6 +++ .../SIARDDK128DatabaseExportModule.java | 6 +++ .../out/output/SIARDDK128ExportModule.java | 6 ++- .../output/SIARDDKDatabaseExportModule.java | 25 +++++---- .../siard/out/output/SIARDDKExportModule.java | 16 +++--- 12 files changed, 134 insertions(+), 35 deletions(-) diff --git a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java index 7d63862a6..de1449ad1 100644 --- a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java +++ b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java @@ -59,7 +59,7 @@ public String escapeSchemaName(String schema) { @Override public String escapeTableName(String table) { - return getStartQuote() + table + getEndQuote(); + return getStartQuote() + StringUtils.strip(table, "\"") + getEndQuote(); } /** diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java index a80f8276f..c6e66d8b0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java @@ -18,6 +18,8 @@ import java.util.List; +import dk.sa.xmlns.diark._1_0.fileindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,11 @@ public SIARDDK1007FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java index d17924986..723712579 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java @@ -17,6 +17,8 @@ package com.databasepreservation.modules.siard.out.metadata; import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,11 @@ public SIARDDK128FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java index 17b8a3ed8..beaf91773 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +57,8 @@ public Object generateXML(DatabaseStructure dbStructure) throws ModuleException String foNbase = baseContainer.getName(count - 1).toString(); // e.g. // AVID.SA.19000.1 - T fileIndexType = createFileIndexTypeInstance(); - List fList = getF(fileIndexType); + JAXBElement fileIndexType = createFileIndexTypeRootInstance(); + List fList = getF(fileIndexType.getValue()); for (Map.Entry entry : md5sums.entrySet()) { @@ -168,6 +169,8 @@ public byte[] addFile(String path) { return digest; } + abstract JAXBElement createFileIndexTypeRootInstance(); + abstract T createFileIndexTypeInstance(); abstract D createFileIndexTypeFInstance(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index 6856cc368..7fee0e106 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -36,13 +36,13 @@ public class SIARDDKMetadataExportStrategy implements MetadataExportStrategy { private static final Logger LOGGER = LoggerFactory.getLogger(SIARDDKMetadataExportStrategy.class); - private SIARDMarshaller siardMarshaller; - private MetadataPathStrategy metadataPathStrategy; - private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; - private SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; - private Map exportModuleArgs; - private LOBsTracker lobsTracker; - private SIARDDKAdapter siarddkAdapter; + protected SIARDMarshaller siardMarshaller; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; + protected SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; + protected Map exportModuleArgs; + protected LOBsTracker lobsTracker; + protected SIARDDKAdapter siarddkAdapter; private Reporter reporter; @@ -197,7 +197,7 @@ private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, } } - private void createLocalSharedFolder(SIARDArchiveContainer container) { + protected void createLocalSharedFolder(SIARDArchiveContainer container) { Path containerPath = container.getPath(); Path localShared = Paths.get("Schemas/localShared"); File folder = containerPath.resolve(localShared).toFile(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java index ec54b1d78..d6efe5353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java @@ -33,4 +33,21 @@ public interface SIARDMarshaller { */ public void marshal(String context, String localeSchemaLocation, String JAXBSchemaLocation, OutputStream writer, Object jaxbElement) throws ModuleException; + + /** + * Generate JAXB Marshaller for writing XML object to the archive. + * + * @param archiveClass + * Siard archive class to give JAXB context + * @param localeSchemaLocation + * The locale location of the XML schema for the metadata file. + * @param JAXBSchemaLocation + * The Marshaller.JAXB_SCHEMA_LOCATION. + * @param writer + * The OutputStream to write to. + * @param jaxbElement + * The JAXB element to marshal. + */ + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException; } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java index fa1d86f47..1a8dfc108 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java @@ -12,9 +12,6 @@ import java.io.OutputStream; import javax.xml.XMLConstants; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -26,6 +23,10 @@ import com.databasepreservation.model.exception.ModuleException; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; + public class StandardSIARDMarshaller implements SIARDMarshaller { private static final String ENCODING = "UTF-8"; @@ -78,4 +79,51 @@ public void marshal(String contextStr, String localeSchemaLocation, String JAXBS throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); } } + + @Override + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException { + + // Set up JAXB marshaller + + JAXBContext context; + try { + context = JAXBContext.newInstance(archiveClass.getPackage().getName(), archiveClass.getClassLoader()); + } catch (JAXBException e) { + throw new ModuleException().withMessage("Error loading JAXBContent").withCause(e); + } + + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema xsdSchema = null; + try { + InputStream in = this.getClass().getResourceAsStream(localeSchemaLocation); + xsdSchema = schemaFactory.newSchema(new StreamSource(in)); + in.close(); + } catch (SAXException e) { + throw new ModuleException() + .withMessage("XSD file has errors: " + getClass().getResource(localeSchemaLocation).getPath()).withCause(e); + } catch (IOException e) { + throw new ModuleException().withMessage("Could not close InputStream").withCause(e); + } + + Marshaller m; + + try { + + m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.setProperty(Marshaller.JAXB_ENCODING, ENCODING); + m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, JAXBSchemaLocation); + + m.setSchema(xsdSchema); + + m.marshal(jaxbElement, writer); + + } catch (JAXBException e) { + if (e.getCause() instanceof SAXParseException) { + LOGGER.error(e.getCause().getMessage()); + } + throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); + } + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java index d251c3570..e775df706 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java @@ -7,6 +7,7 @@ */ package com.databasepreservation.modules.siard.out.output; +import com.databasepreservation.modules.siard.bindings.siard_dk_1007.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; /** @@ -23,4 +24,9 @@ public SIARDDK1007DatabaseExportModule(SIARDDKExportModule siarddkExportModule) String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX; } + + @Override + Class getJAXBContextClass() { + return SiardDiark.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java index 334aa2017..ab8b88353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java @@ -7,6 +7,7 @@ */ package com.databasepreservation.modules.siard.out.output; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; /** @@ -23,4 +24,9 @@ public SIARDDK128DatabaseExportModule(SIARDDKExportModule siarddkExportModule) { String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX_128; } + + @Override + Class getJAXBContextClass() { + return SiardDiark.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java index 0c70f2856..9d1876645 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java @@ -14,15 +14,17 @@ package com.databasepreservation.modules.siard.out.output; +import java.util.Map; + import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter; import com.databasepreservation.modules.siard.common.path.SIARDDK128MetadataPathStrategy; import com.databasepreservation.modules.siard.common.path.SIARDDKMetadataPathStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128DocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128FileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDDK128MetadataExportStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKDocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKMetadataExportStrategy; -import java.util.Map; /** * @author António Lindo @@ -51,7 +53,7 @@ SIARDDKMetadataPathStrategy createSIARDDKMetadataPathStrategyInstance() { @Override SIARDDKMetadataExportStrategy createSIARDDKMetadataExportStrategyInstance() { - return new SIARDDKMetadataExportStrategy(this, new SIARDDK128Adapter()); + return new SIARDDK128MetadataExportStrategy(this, new SIARDDK128Adapter()); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index e9a85f2cc..031bceb1a 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -7,16 +7,6 @@ */ package com.databasepreservation.modules.siard.out.output; -import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; -import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; import java.io.OutputStream; @@ -25,6 +15,17 @@ import java.nio.file.attribute.BasicFileAttributes; import java.util.Map; +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; + /** * @author Andreas Kring * @@ -117,7 +118,7 @@ public void finishDatabase() throws ModuleException { OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(siarddkExportModule.getMainContainer(), path, siarddkExportModule.getWriteStrategy()); - siardMarshaller.marshal(getJAXBContext(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), + siardMarshaller.marshal(getJAXBContextClass(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/fileIndex.xsd", writer, SIARDDKFileIndexFileStrategy.generateXML(null)); @@ -129,4 +130,6 @@ public void finishDatabase() throws ModuleException { } abstract String getJAXBContext(); + + abstract Class getJAXBContextClass(); } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java index 7fd8d54d0..85c7e2dc6 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java @@ -35,14 +35,14 @@ * */ public abstract class SIARDDKExportModule { - private MetadataExportStrategy metadataExportStrategy; - private SIARDArchiveContainer mainContainer; - private ContentExportStrategy contentExportStrategy; - private WriteStrategy writeStrategy; - private ContentPathExportStrategy contentPathExportStrategy; - private MetadataPathStrategy metadataPathStrategy; - private SIARDMarshaller siardMarshaller; - private LOBsTracker lobsTracker; + protected MetadataExportStrategy metadataExportStrategy; + protected SIARDArchiveContainer mainContainer; + protected ContentExportStrategy contentExportStrategy; + protected WriteStrategy writeStrategy; + protected ContentPathExportStrategy contentPathExportStrategy; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDMarshaller siardMarshaller; + protected LOBsTracker lobsTracker; private Map exportModuleArgs; private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; From aff3f1ab9bd9a4e7afba4a46de413f37852b500f Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 15 May 2026 14:34:24 +0100 Subject: [PATCH 04/37] WIP - some SIARDDK and data type export fixes --- .../databasepreservation/modules/jdbc/out/JDBCExportModule.java | 2 +- .../siard/in/metadata/SIARDDK128MetadataImportStrategy.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java index 9d029d73c..92a7b009e 100644 --- a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java +++ b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java @@ -684,7 +684,7 @@ protected void handleSimpleTypeStringDataCell(String data, PreparedStatement ps, protected void handleSimpleTypeNumericExactDataCell(String data, PreparedStatement ps, int index, Cell cell, ColumnStructure column) throws SQLException { - if (data != null) { + if (data != null && !data.isEmpty()) { BigDecimal bd = new BigDecimal(data); ps.setBigDecimal(index, bd); } else { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java index c02344dd1..25bdeb1fe 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java @@ -323,7 +323,7 @@ private TableStructure createContextDocumentationTable() throws ModuleException virtualTable.setRows(contextDocumentationIndex.getDocument().size()); virtualTable.setColumns(createContextDocumentsTableColumns()); virtualTable.setPrimaryKey(createVirtualPrimaryKey( - SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DID)); + SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DOCUMENT_ID)); return virtualTable; } } catch (FileNotFoundException e) { From 2c4babb30f085d89176c47925d76177b9c28a31a Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Mon, 18 May 2026 09:05:23 +0100 Subject: [PATCH 05/37] Add SIARDDK128MetadataExportStrategy --- .../SIARDDK128MetadataExportStrategy.java | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java new file mode 100644 index 000000000..d2a4429e0 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -0,0 +1,109 @@ +package com.databasepreservation.modules.siard.out.metadata; + +import java.io.IOException; +import java.io.OutputStream; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.structure.DatabaseStructure; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; +import com.databasepreservation.modules.siard.common.adapters.SIARDDKAdapter; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; +import com.databasepreservation.modules.siard.out.write.WriteStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK128MetadataExportStrategy extends SIARDDKMetadataExportStrategy { + + public SIARDDK128MetadataExportStrategy(SIARDDKExportModule siarddkExportModule, SIARDDKAdapter siarddkAdapter) { + super(siarddkExportModule, siarddkAdapter); + } + + @Override + public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + // TO-DO: Refactor this into one method in class that can be used by + // SIARDDKDatabaseExportModule also + + // Generate tableIndex.xml + + try { + IndexFileStrategy tableIndexFileStrategy = new SIARDDKTableIndexFileStrategy(lobsTracker, siarddkAdapter); + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal("com.databasepreservation.modules.siard.bindings.siard_dk_128", + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.TABLE_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/tableIndex.xsd", writer, + tableIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing tableIndex.xml to the archive.").withCause(e); + } + + // Generate archiveIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.ARCHIVE_INDEX) != null) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.ARCHIVE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy archiveIndexFileStrategy = new CommandLineIndexFileStrategy(SIARDDKConstants.ARCHIVE_INDEX, + exportModuleArgs, writer, metadataPathStrategy); + archiveIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing archiveIndex.xml to the archive").withCause(e); + } + } + + // Generate contextDocumentationIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX) != null) { + try { + + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy contextDocumentationIndexFileStrategy = new CommandLineIndexFileStrategy( + SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, exportModuleArgs, writer, metadataPathStrategy); + contextDocumentationIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing contextDocumentationIndex.xml to the archive") + .withCause(e); + } + } + + if (lobsTracker.getLOBsCount() > 0) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal(SIARDDKConstants.JAXB_CONTEXT_DOCINDEX, + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/docIndex.xsd", writer, + SIARDDKDocIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing docIndex.xml to the archive.").withCause(e); + } + } + + createLocalSharedFolder(outputContainer); + } +} From e821cad654bf32a1045f88d873d4a32e87f916ab Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 22 May 2026 16:03:24 +0100 Subject: [PATCH 06/37] Fix SIARDDK128 docindex marshalling --- .../metadata/SIARDDK1007DocIndexFileStrategy.java | 7 +++++++ .../out/metadata/SIARDDK128DocIndexFileStrategy.java | 12 ++++++++++-- .../out/metadata/SIARDDKDocIndexFileStrategy.java | 9 ++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java index ae10da9ae..04720fbf4 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java @@ -12,6 +12,8 @@ import dk.sa.xmlns.diark._1_0.docindex.DocIndexType; import dk.sa.xmlns.diark._1_0.docindex.DocumentType; +import dk.sa.xmlns.diark._1_0.docindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +25,11 @@ public SIARDDK1007DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java index c57ae786c..259518690 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java @@ -7,11 +7,14 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.math.BigInteger; +import java.util.List; + import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocumentType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; -import java.math.BigInteger; -import java.util.List; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +26,11 @@ public SIARDDK128DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java index 62e393839..c0df07692 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.structure.DatabaseStructure; +import jakarta.xml.bind.JAXBElement; /** * @author Andreas Kring @@ -19,10 +20,10 @@ */ public abstract class SIARDDKDocIndexFileStrategy implements IndexFileStrategy { - private T docIndex; + private JAXBElement docIndex; public SIARDDKDocIndexFileStrategy() { - docIndex = createDocIndexTypeInstance(); + docIndex = createDocIndexTypeRootInstance(); } /* @@ -70,11 +71,13 @@ public D addDoc(int dID, int pID, int mID, int docCollectionNumber, String oFn, setGmlXsd(doc, gmlXsd); } - getDoc(docIndex).add(doc); + getDoc(docIndex.getValue()).add(doc); return doc; } + abstract JAXBElement createDocIndexTypeRootInstance(); + abstract T createDocIndexTypeInstance(); abstract D createDocumentTypeInstance(); From f9bcba3a084616cd62671db5e605c830cb7dfbd8 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Wed, 27 May 2026 09:08:09 +0100 Subject: [PATCH 07/37] Fix JAXB context for DocIndex in SIARD DK 128 --- .../siard/out/metadata/SIARDDK128MetadataExportStrategy.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java index d2a4429e0..4a0b82c5f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -5,6 +5,7 @@ import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.structure.DatabaseStructure; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.common.adapters.SIARDDKAdapter; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; @@ -90,8 +91,7 @@ public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContaine String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX); OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); - siardMarshaller.marshal(SIARDDKConstants.JAXB_CONTEXT_DOCINDEX, - metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), + siardMarshaller.marshal(DocIndexType.class, metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/docIndex.xsd", writer, SIARDDKDocIndexFileStrategy.generateXML(dbStructure)); From 21612680c001da4411fd22808c538bb2a92572bb Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Tue, 2 Jun 2026 09:29:40 +0100 Subject: [PATCH 08/37] Implement asynchronous LOB conversion with HTTP service and enhance BinaryCell to include mimeType --- .../model/data/BinaryCell.java | 11 + .../content/SIARDDKContentExportStrategy.java | 5 +- .../output/SIARDDKDatabaseExportModule.java | 110 +++++++++- .../services/conversion/ConversionResult.java | 9 + .../conversion/HttpLobConversionService.java | 204 ++++++++++++++++++ .../conversion/JobStatusResponse.java | 7 + .../conversion/JobSubmissionResponse.java | 7 + .../conversion/LobConversionService.java | 8 + .../services/conversion/TempFileTracker.java | 33 +++ 9 files changed, 391 insertions(+), 3 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java index 8b9149f11..63f8fbf3b 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java @@ -28,6 +28,7 @@ public class BinaryCell extends Cell implements InputStreamProvider { private InputStreamProvider inputStreamProvider; private String file; private long length; + private String mimeType; /** * Creates a binary cell. This binary cell will mostly just be a wrapper around @@ -75,6 +76,12 @@ public BinaryCell(String id, InputStreamProvider inputStreamProvider) { this.inputStreamProvider = inputStreamProvider; } + public BinaryCell(String id, InputStreamProvider inputStreamProvider, String mimeType) { + super(id); + this.inputStreamProvider = inputStreamProvider; + this.mimeType = mimeType; + } + /** * Creates a binary cell. This binary cell is a wrapper around a * ProvidesInputStream object (whilst also providing Cell functionality). @@ -128,4 +135,8 @@ public String getFile() { public long getLength() { return length; } + + public String getMimeType() { + return mimeType; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 47163f158..9bb53fe9d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -331,7 +331,7 @@ public Row tableRow(Row row) throws ModuleException { InputStream is = new BufferedInputStream(binaryCell.createInputStream()); // Removed because TIKA was a security vulnerability and this feature was not // needed/not fully implemented (see #341) - String mimeType = "unsupported"; + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; IOUtils.closeQuietly(is); // Archive BLOB - simultaneous writing always supported for @@ -357,7 +357,8 @@ public Row tableRow(Row row) throws ModuleException { // Create new FileIndexFileStrategy // Write the BLOB - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), + writeStrategy); InputStream in = blob.getInputStreamProvider().createInputStream(); IOUtils.copy(in, out); IOUtils.closeQuietly(in); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 031bceb1a..b9398be96 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -9,22 +9,39 @@ import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.LinkedList; +import java.util.List; import java.util.Map; +import java.util.Queue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.common.io.providers.PathInputStreamProvider; +import com.databasepreservation.model.data.BinaryCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.model.data.Row; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; +import com.databasepreservation.modules.siard.services.conversion.ConversionResult; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; +import com.databasepreservation.modules.siard.services.conversion.LobConversionService; +import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; /** * @author Andreas Kring @@ -35,6 +52,12 @@ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { private SIARDDKExportModule siarddkExportModule; private static final Logger logger = LoggerFactory.getLogger(SIARDDKDatabaseExportModule.class); + private ExecutorService executorService; + private Queue> pendingRowsQueue; + private TempFileTracker tempFileTracker; + private LobConversionService conversionService; + private static final int MAX_QUEUE_SIZE = 100; + public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { super(siarddkExportModule.getContentExportStrategy(), siarddkExportModule.getMainContainer(), siarddkExportModule.getWriteStrategy(), siarddkExportModule.getMetadataExportStrategy(), null); @@ -46,6 +69,12 @@ public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { public void initDatabase() throws ModuleException { super.initDatabase(); + this.tempFileTracker = new TempFileTracker(); + String apiEndpoint = "http://localhost:8080"; + String targetFormat = "image/tiff"; + this.conversionService = new HttpLobConversionService(apiEndpoint, targetFormat, this.tempFileTracker); + this.executorService = Executors.newVirtualThreadPerTaskExecutor(); + // Get docID info from the command line and add these to the LOBsTracker Path pathToArchive = siarddkExportModule.getMainContainer().getPath(); @@ -83,8 +112,47 @@ public void initDatabase() throws ModuleException { } } + @Override + public void handleDataOpenTable(String tableId) throws ModuleException { + // Prepare the FIFO queue for the new table + this.pendingRowsQueue = new LinkedList<>(); + super.handleDataOpenTable(tableId); + } + + @Override + public void handleDataRow(Row row) throws ModuleException { + // 1. Submit the row conversion to the Virtual Thread + Callable conversionTask = () -> processRow(row); + Future futureRow = executorService.submit(conversionTask); + pendingRowsQueue.add(futureRow); + + logger.debug("Submitted row for asynchronous processing. Current queue size: {}", pendingRowsQueue.size()); + // 2. Backpressure: Wait for the queue to drain if it reaches the limit + while (pendingRowsQueue.size() >= MAX_QUEUE_SIZE) { + logger.debug("Pending rows queue has reached the maximum size of {}. Waiting for the oldest task to complete...", + MAX_QUEUE_SIZE); + drainHeadAndExport(); + } + } + + @Override + public void handleDataCloseTable(String tableId) throws ModuleException { + // Process and export all remaining rows in the queue + while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { + drainHeadAndExport(); + } + super.handleDataCloseTable(tableId); + } + @Override public void finishDatabase() throws ModuleException { + if (executorService != null && !executorService.isShutdown()) { + executorService.shutdown(); + } + if (tempFileTracker != null) { + tempFileTracker.cleanupAll(); + } + super.finishDatabase(); // Write ContextDocumentation to archive @@ -118,7 +186,8 @@ public void finishDatabase() throws ModuleException { OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(siarddkExportModule.getMainContainer(), path, siarddkExportModule.getWriteStrategy()); - siardMarshaller.marshal(getJAXBContextClass(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), + siardMarshaller.marshal(getJAXBContextClass(), + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/fileIndex.xsd", writer, SIARDDKFileIndexFileStrategy.generateXML(null)); @@ -132,4 +201,43 @@ public void finishDatabase() throws ModuleException { abstract String getJAXBContext(); abstract Class getJAXBContextClass(); + + private void drainHeadAndExport() throws ModuleException { + Future oldestFuture = pendingRowsQueue.poll(); + if (oldestFuture != null) { + try { + logger.debug("Waiting for the oldest row conversion task to complete. Remaining queue size after polling: {}", + pendingRowsQueue.size()); + Row processedRow = oldestFuture.get(); + + logger.debug("Oldest row conversion task completed. Exporting row with ID: {}", processedRow.getIndex()); + + super.handleDataRow(processedRow); + } catch (InterruptedException | ExecutionException e) { + oldestFuture.cancel(true); + throw new ModuleException().withMessage("Error processing row conversion task").withCause(e); + } + } + } + + private Row processRow(Row row) throws Exception { + logger.debug("Processing row with ID: {} in thread: {}", row.getIndex(), Thread.currentThread().getName()); + List cells = row.getCells(); + + for (int i = 0; i < cells.size(); i++) { + Cell cell = cells.get(i); + + if (cell instanceof BinaryCell binCell) { + try (InputStream originalStream = binCell.createInputStream()) { + ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); + Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFile()), + "image/tiff"); + cells.set(i, newCell); + } + } + } + row.setCells(cells); + logger.debug("Completed processing row with ID: {}", row.getIndex()); + return row; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java new file mode 100644 index 000000000..4b48a9bc5 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java @@ -0,0 +1,9 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Path; + +/** + * @author Gabriel Barros + */ +public record ConversionResult(Path convertedFile, Path reportFile) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java new file mode 100644 index 000000000..b85cf2d89 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -0,0 +1,204 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.Arrays; +import java.util.Collections; +import java.util.Random; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Handles communication with the external format conversion REST API. Uses + * asynchronous polling and in-memory multipart streaming to process large + * objects efficiently. + */ +public class HttpLobConversionService implements LobConversionService { + + private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); + + private static final int MAX_POLLING_ATTEMPTS = 300; + private static final int MAX_NETWORK_RETRIES = 3; + private static final int BASE_POLLING_INTERVAL_MS = 2000; + + private final HttpClient httpClient; + private final String baseUrl; + private final String targetFormat; + private final ObjectMapper objectMapper; + private final TempFileTracker fileTracker; + private final Random random = new Random(); + + public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTracker fileTracker) { + this.baseUrl = baseUrl; + this.targetFormat = targetFormat; + this.fileTracker = fileTracker; + + this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(30)).build(); + } + + @Override + public ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception { + String jobId = submitJob(cellId, inputStream); + waitForCompletion(cellId, jobId); + return downloadAndExtractResult(cellId, jobId); + } + + /** + * Submits the job using a SequenceInputStream to stream the multipart request + * directly, avoiding intermediate disk writes for the payload. + */ + private String submitJob(String cellId, InputStream inputStream) throws Exception { + String boundary = "DbptkBoundary" + System.currentTimeMillis(); + + String header = "--" + boundary + "\r\n" + "Content-Disposition: form-data; name=\"targetFormat\"\r\n\r\n" + + targetFormat + "\r\n" + "--" + boundary + "\r\n" + + "Content-Disposition: form-data; name=\"file\"; filename=\"lob_" + cellId + ".bin\"\r\n" + + "Content-Type: application/octet-stream\r\n\r\n"; + + String footer = "\r\n--" + boundary + "--\r\n"; + + InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); + InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); + + // Chains the header, actual LOB data, and footer without loading the LOB into + // memory + InputStream multipartStream = new SequenceInputStream( + Collections.enumeration(Arrays.asList(headerStream, inputStream, footerStream))); + + HttpRequest submitRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs")) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); + + // Not using executeWithRetry here because the InputStream is consumed and + // cannot be trivially reset. + HttpResponse submitResponse = httpClient.send(submitRequest, BodyHandlers.ofString()); + + if (submitResponse.statusCode() >= 400) { + throw new RuntimeException("Failed to submit LOB for cell " + cellId + ": " + submitResponse.body()); + } + + JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); + return job.id(); + } + + /** + * Polls the job status API until completion or timeout. Includes jitter to + * prevent thundering herd. + */ + private void waitForCompletion(String cellId, String jobId) throws Exception { + for (int i = 0; i < MAX_POLLING_ATTEMPTS; i++) { + HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); + + HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), + MAX_NETWORK_RETRIES); + JobStatusResponse status = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); + + switch (status.status().toUpperCase()) { + case "COMPLETED", "DONE", "SUCCESS" -> { + return; + } + case "FAILED", "ERROR", "EVICTED" -> throw new RuntimeException("Server failed to convert cell: " + cellId); + default -> { + long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); // Jittering + Thread.sleep(sleepTime); + } + } + } + throw new RuntimeException("Timeout after waiting for conversion of cell: " + cellId); + } + + /** + * Downloads the resulting ZIP and extracts its contents. + */ + private ConversionResult downloadAndExtractResult(String cellId, String jobId) throws Exception { + Path tempZipFile = Files.createTempFile("siarddk_conv_" + cellId + "_", ".zip"); + fileTracker.track(tempZipFile); + + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); + + executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); + + return extractZipContents(cellId, tempZipFile); + } + + private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { + Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); + fileTracker.trackDir(extractionDir); + + Path convertedLob = null; + Path reportFile = null; + + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tempZipFile.toFile()))) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()); + + // Zip Slip vulnerability prevention + if (!extractedFilePath.normalize().startsWith(extractionDir)) { + throw new SecurityException("Corrupted ZIP entry: " + zipEntry.getName()); + } + + if (!zipEntry.isDirectory()) { + Files.copy(zis, extractedFilePath); + + if (zipEntry.getName().toLowerCase().contains("report")) { + reportFile = extractedFilePath; + } else { + convertedLob = extractedFilePath; + } + } + } + } + + if (convertedLob == null || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (LOB + Report) for cell: " + cellId); + } + + return new ConversionResult(convertedLob, reportFile); + } + + /** + * Enforces resilient networking via exponential backoff. Intended exclusively + * for idempotent requests (e.g., GET). + */ + private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, + int maxRetries) throws Exception { + Exception lastException = null; + + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + return httpClient.send(request, responseBodyHandler); + } catch (IOException e) { + lastException = e; + log.warn("Attempt {} failed for {}: {}", attempt, request.uri(), e.getMessage()); + + if (attempt == maxRetries) + break; + Thread.sleep((long) Math.pow(2, attempt) * 1000); // Exponential backoff: 2s, 4s, 8s... + } + } + throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); + } +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java new file mode 100644 index 000000000..79f8551c3 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public record JobStatusResponse(String status) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java new file mode 100644 index 000000000..2abfaf6dd --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public record JobSubmissionResponse(String id) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java new file mode 100644 index 000000000..038e779b4 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.InputStream; + +public interface LobConversionService { + ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception; +} + diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java new file mode 100644 index 000000000..b138a6ace --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java @@ -0,0 +1,33 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class TempFileTracker { + private static final Logger LOGGER = LoggerFactory.getLogger(TempFileTracker.class); + private final Queue trackedFiles = new ConcurrentLinkedQueue<>(); + + public void track(Path path) { + trackedFiles.add(path); + } + + public void trackDir(Path dirPath) { + trackedFiles.add(dirPath); + } + + public void cleanupAll() { + for (Path path : trackedFiles) { + try { + Files.deleteIfExists(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary file/directory: " + path, e); + } + } + trackedFiles.clear(); + } +} \ No newline at end of file From 681eb33433e3db353f9ba534adf157b2e865828b Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Tue, 2 Jun 2026 16:21:02 +0100 Subject: [PATCH 09/37] Enhance HttpLobConversionService and SIARDDKDatabaseExportModule for improved asynchronous LOB processing and error handling --- .../output/SIARDDKDatabaseExportModule.java | 188 ++++++++++++++---- .../conversion/HttpLobConversionService.java | 69 ++++--- .../services/conversion/TempFileTracker.java | 76 ++++++- 3 files changed, 252 insertions(+), 81 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index b9398be96..ce0fa1e01 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -14,15 +14,18 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; -import java.util.LinkedList; +import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import org.apache.commons.io.FileUtils; import org.slf4j.Logger; @@ -44,20 +47,34 @@ import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; /** + * Handles database export pipeline matching SIARD-DK compliance, incorporating + * high-throughput asynchronous LOB streaming conversion. + * * @author Andreas Kring * */ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { - private SIARDDKExportModule siarddkExportModule; + private final SIARDDKExportModule siarddkExportModule; private static final Logger logger = LoggerFactory.getLogger(SIARDDKDatabaseExportModule.class); private ExecutorService executorService; - private Queue> pendingRowsQueue; private TempFileTracker tempFileTracker; private LobConversionService conversionService; private static final int MAX_QUEUE_SIZE = 100; + // Resilient Pipeline architecture attributes + private BlockingQueue> pendingRowsQueue; + private ExecutorService writerExecutor; + private Future writerTask; + private final AtomicReference writerError = new AtomicReference<>(); + + /** + * Data context tuple to link rows with their transient extracted disk paths. + */ + private record ProcessedRowContext(Row row, List transientPaths) { + } + public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { super(siarddkExportModule.getContentExportStrategy(), siarddkExportModule.getMainContainer(), siarddkExportModule.getWriteStrategy(), siarddkExportModule.getMetadataExportStrategy(), null); @@ -70,10 +87,11 @@ public void initDatabase() throws ModuleException { super.initDatabase(); this.tempFileTracker = new TempFileTracker(); - String apiEndpoint = "http://localhost:8080"; + String apiEndpoint = "http://localhost:8087"; String targetFormat = "image/tiff"; this.conversionService = new HttpLobConversionService(apiEndpoint, targetFormat, this.tempFileTracker); this.executorService = Executors.newVirtualThreadPerTaskExecutor(); + this.writerExecutor = Executors.newSingleThreadExecutor(); // Dedicated single-thread pipeline consumer // Get docID info from the command line and add these to the LOBsTracker @@ -114,33 +132,22 @@ public void initDatabase() throws ModuleException { @Override public void handleDataOpenTable(String tableId) throws ModuleException { - // Prepare the FIFO queue for the new table - this.pendingRowsQueue = new LinkedList<>(); + logger.debug("Opening table '{}'. Initializing asynchronous pipeline...", tableId); + this.pendingRowsQueue = new LinkedBlockingQueue<>(MAX_QUEUE_SIZE); + this.writerError.set(null); + startAsyncWriter(); super.handleDataOpenTable(tableId); } @Override public void handleDataRow(Row row) throws ModuleException { - // 1. Submit the row conversion to the Virtual Thread - Callable conversionTask = () -> processRow(row); - Future futureRow = executorService.submit(conversionTask); - pendingRowsQueue.add(futureRow); - - logger.debug("Submitted row for asynchronous processing. Current queue size: {}", pendingRowsQueue.size()); - // 2. Backpressure: Wait for the queue to drain if it reaches the limit - while (pendingRowsQueue.size() >= MAX_QUEUE_SIZE) { - logger.debug("Pending rows queue has reached the maximum size of {}. Waiting for the oldest task to complete...", - MAX_QUEUE_SIZE); - drainHeadAndExport(); - } + enqueueRow(row); } @Override public void handleDataCloseTable(String tableId) throws ModuleException { - // Process and export all remaining rows in the queue - while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { - drainHeadAndExport(); - } + logger.debug("Closing table '{}'. Draining remaining items in the pipeline...", tableId); + stopAsyncWriter(); super.handleDataCloseTable(tableId); } @@ -149,6 +156,9 @@ public void finishDatabase() throws ModuleException { if (executorService != null && !executorService.isShutdown()) { executorService.shutdown(); } + if (writerExecutor != null && !writerExecutor.isShutdown()) { + writerExecutor.shutdown(); + } if (tempFileTracker != null) { tempFileTracker.cleanupAll(); } @@ -195,34 +205,111 @@ public void finishDatabase() throws ModuleException { } catch (IOException e) { throw new ModuleException().withMessage("Error writing fileIndex to the archive.").withCause(e); } - } - abstract String getJAXBContext(); + /** + * Starts the sequential pipeline background consumer thread. + */ + private void startAsyncWriter() { + this.writerTask = writerExecutor.submit(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + Future future = pendingRowsQueue.take(); // Enforces strict sequential order + ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary - abstract Class getJAXBContextClass(); + super.handleDataRow(context.row()); - private void drainHeadAndExport() throws ModuleException { - Future oldestFuture = pendingRowsQueue.poll(); - if (oldestFuture != null) { + // Alleviate disk pressure by wiping extracted structures instantly after XML + // writing + cleanupTransientPaths(context.transientPaths()); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Pipeline consumer thread was interrupted and is shutting down."); + } catch (ExecutionException e) { + logger.error("A background conversion task failed critically: {}", e.getCause().getMessage()); + writerError.set(e.getCause()); + } catch (Exception e) { + logger.error("An unexpected error occurred during sequential writing.", e); + writerError.set(e); + } + }); + } + + /** + * Gracefully drains the remaining queue, safely stops the consumer and monitors + * failures. + */ + private void stopAsyncWriter() throws ModuleException { + while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { + if (writerError.get() != null) { + break; + } try { - logger.debug("Waiting for the oldest row conversion task to complete. Remaining queue size after polling: {}", - pendingRowsQueue.size()); - Row processedRow = oldestFuture.get(); + TimeUnit.MILLISECONDS.sleep(20); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } - logger.debug("Oldest row conversion task completed. Exporting row with ID: {}", processedRow.getIndex()); + if (writerTask != null) { + writerTask.cancel(true); + } - super.handleDataRow(processedRow); - } catch (InterruptedException | ExecutionException e) { - oldestFuture.cancel(true); - throw new ModuleException().withMessage("Error processing row conversion task").withCause(e); + // Purge and cancel any remaining futures to avoid thread and resource leakage + if (pendingRowsQueue != null) { + for (Future future : pendingRowsQueue) { + future.cancel(true); + try { + if (future.isDone() && !future.isCancelled()) { + cleanupTransientPaths(future.get().transientPaths()); + } + } catch (Exception ignored) { + } } + pendingRowsQueue.clear(); + } + + if (writerError.get() != null) { + throw new ModuleException().withMessage("Row writing pipeline aborted").withCause(writerError.get()); + } + } + + /** + * Pushes rows into the pipeline, throwing fast if the writer task fails, and + * using non-deadlocking backpressure. + */ + private void enqueueRow(Row row) throws ModuleException { + if (writerError.get() != null) { + throw new ModuleException().withMessage("Pipeline execution halted due to previous background failure") + .withCause(writerError.get()); + } + + Callable conversionTask = () -> processRowAsync(row); + Future future = executorService.submit(conversionTask); + + try { + // Prevents deadlocks if the consumer thread crashes while queue is maxed out + while (!pendingRowsQueue.offer(future, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) { + future.cancel(true); + throw new ModuleException().withMessage("Pipeline halted while enqueuing row").withCause(writerError.get()); + } + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + future.cancel(true); + throw new ModuleException().withMessage("Row enqueuing process was interrupted").withCause(e); } } - private Row processRow(Row row) throws Exception { - logger.debug("Processing row with ID: {} in thread: {}", row.getIndex(), Thread.currentThread().getName()); + /** + * Processes row columns concurrently on Virtual Threads mapping extracted files + * for lifecycle control. + */ + private ProcessedRowContext processRowAsync(Row row) throws Exception { List cells = row.getCells(); + List transientPaths = new ArrayList<>(); for (int i = 0; i < cells.size(); i++) { Cell cell = cells.get(i); @@ -233,11 +320,28 @@ private Row processRow(Row row) throws Exception { Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFile()), "image/tiff"); cells.set(i, newCell); + + // Track extracted parts to clean them individually later + transientPaths.add(result.convertedFile()); + transientPaths.add(result.reportFile()); + transientPaths.add(result.convertedFile().getParent()); // directory container } } } row.setCells(cells); - logger.debug("Completed processing row with ID: {}", row.getIndex()); - return row; + return new ProcessedRowContext(row, transientPaths); + } + + private void cleanupTransientPaths(List paths) { + if (paths == null || tempFileTracker == null) { + return; + } + for (Path path : paths) { + tempFileTracker.deleteEarly(path); + } } -} + + abstract String getJAXBContext(); + + abstract Class getJAXBContextClass(); +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index b85cf2d89..4f143fb87 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -31,14 +31,16 @@ * Handles communication with the external format conversion REST API. Uses * asynchronous polling and in-memory multipart streaming to process large * objects efficiently. + * + * @author Gabriel Barros */ public class HttpLobConversionService implements LobConversionService { private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); - private static final int MAX_POLLING_ATTEMPTS = 300; private static final int MAX_NETWORK_RETRIES = 3; private static final int BASE_POLLING_INTERVAL_MS = 2000; + private static final int MAX_POLLING_ATTEMPTS = 600; // ~20 minutes maximum wait per file before assuming Zombie Job private final HttpClient httpClient; private final String baseUrl; @@ -51,23 +53,18 @@ public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTra this.baseUrl = baseUrl; this.targetFormat = targetFormat; this.fileTracker = fileTracker; - this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - - this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(30)).build(); + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60)).build(); } @Override public ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception { + log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); return downloadAndExtractResult(cellId, jobId); } - /** - * Submits the job using a SequenceInputStream to stream the multipart request - * directly, avoiding intermediate disk writes for the payload. - */ private String submitJob(String cellId, InputStream inputStream) throws Exception { String boundary = "DbptkBoundary" + System.currentTimeMillis(); @@ -81,8 +78,6 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); - // Chains the header, actual LOB data, and footer without loading the LOB into - // memory InputStream multipartStream = new SequenceInputStream( Collections.enumeration(Arrays.asList(headerStream, inputStream, footerStream))); @@ -90,57 +85,72 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); - // Not using executeWithRetry here because the InputStream is consumed and - // cannot be trivially reset. HttpResponse submitResponse = httpClient.send(submitRequest, BodyHandlers.ofString()); if (submitResponse.statusCode() >= 400) { - throw new RuntimeException("Failed to submit LOB for cell " + cellId + ": " + submitResponse.body()); + log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), + submitResponse.body()); + throw new RuntimeException("Failed to submit LOB for cell " + cellId); } JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); + log.debug("Successfully dispatched cell {}. Assigned Job ID: {}", cellId, job.id()); return job.id(); } - /** - * Polls the job status API until completion or timeout. Includes jitter to - * prevent thundering herd. - */ private void waitForCompletion(String cellId, String jobId) throws Exception { - for (int i = 0; i < MAX_POLLING_ATTEMPTS; i++) { - HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); + log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); + for (int attempts = 1; attempts <= MAX_POLLING_ATTEMPTS; attempts++) { + HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); + JobStatusResponse status = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); switch (status.status().toUpperCase()) { case "COMPLETED", "DONE", "SUCCESS" -> { + log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); return; } - case "FAILED", "ERROR", "EVICTED" -> throw new RuntimeException("Server failed to convert cell: " + cellId); + case "FAILED", "ERROR", "EVICTED" -> { + log.error("API reported terminal failure for Job {} (Cell {})", jobId, cellId); + throw new RuntimeException("Server failed to convert cell: " + cellId); + } default -> { - long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); // Jittering + if (attempts % 30 == 0) { + log.warn("Job {} (Cell {}) is taking unusually long. Current status: {}. Attempt: {}/{}", jobId, cellId, + status.status(), attempts, MAX_POLLING_ATTEMPTS); + } + long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); Thread.sleep(sleepTime); } } } + log.error("Zombie Job detected. API failed to resolve Job {} (Cell {}) within the maximum polling threshold.", + jobId, cellId); throw new RuntimeException("Timeout after waiting for conversion of cell: " + cellId); } /** - * Downloads the resulting ZIP and extracts its contents. + * Downloads the resulting ZIP and extracts its contents, freeing the ZIP file + * immediately after. */ private ConversionResult downloadAndExtractResult(String cellId, String jobId) throws Exception { Path tempZipFile = Files.createTempFile("siarddk_conv_" + cellId + "_", ".zip"); fileTracker.track(tempZipFile); - HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) - .GET().build(); + try { + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); - executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); + executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); - return extractZipContents(cellId, tempZipFile); + return extractZipContents(cellId, tempZipFile); + } finally { + // Free disk space immediately after extraction + fileTracker.deleteEarly(tempZipFile); + } } private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { @@ -155,7 +165,6 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr while ((zipEntry = zis.getNextEntry()) != null) { Path extractedFilePath = extractionDir.resolve(zipEntry.getName()); - // Zip Slip vulnerability prevention if (!extractedFilePath.normalize().startsWith(extractionDir)) { throw new SecurityException("Corrupted ZIP entry: " + zipEntry.getName()); } @@ -179,10 +188,6 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr return new ConversionResult(convertedLob, reportFile); } - /** - * Enforces resilient networking via exponential backoff. Intended exclusively - * for idempotent requests (e.g., GET). - */ private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, int maxRetries) throws Exception { Exception lastException = null; @@ -196,7 +201,7 @@ private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.B if (attempt == maxRetries) break; - Thread.sleep((long) Math.pow(2, attempt) * 1000); // Exponential backoff: 2s, 4s, 8s... + Thread.sleep((long) Math.pow(2, attempt) * 1000); } } throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java index b138a6ace..ae82d21e0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java @@ -1,33 +1,95 @@ package com.databasepreservation.modules.siard.services.conversion; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Comparator; import java.util.Queue; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Tracks and manages the lifecycle of temporary files and directories created + * during the LOB conversion process. + * + * @author Gabriel Barros + */ public class TempFileTracker { private static final Logger LOGGER = LoggerFactory.getLogger(TempFileTracker.class); - private final Queue trackedFiles = new ConcurrentLinkedQueue<>(); + private final Queue trackedPaths = new ConcurrentLinkedQueue<>(); + /** + * Registers a file or directory path to be tracked for final cleanup. * @param + * path The path to track. + */ public void track(Path path) { - trackedFiles.add(path); + if (path != null) { + trackedPaths.add(path); + } } + /** + * Registers a directory path to be tracked for final cleanup. * @param dirPath + * The directory path to track. + */ public void trackDir(Path dirPath) { - trackedFiles.add(dirPath); + track(dirPath); + } + + /** + * Deletes a tracked path immediately (including non-empty directories) and + * removes it from the tracking queue to free resources early. * @param path The + * path to delete immediately. + */ + public void deleteEarly(Path path) { + if (path == null) { + return; + } + try { + deleteRecursively(path); + trackedPaths.remove(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary path early: " + path, e); + } } + /** + * Deletes all remaining tracked files and directories comprehensively, ensuring + * recursive cleanup of nested content. + */ public void cleanupAll() { - for (Path path : trackedFiles) { + for (Path path : trackedPaths) { try { - Files.deleteIfExists(path); + deleteRecursively(path); } catch (Exception e) { - LOGGER.warn("Unable to delete temporary file/directory: " + path, e); + LOGGER.warn("Unable to delete temporary file/directory during bulk cleanup: " + path, e); + } + } + trackedPaths.clear(); + } + + /** + * Helper method to perform safe recursive deletion of paths and directories. + */ + private void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + if (Files.isDirectory(path)) { + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException e) { + LOGGER.warn("Failed to delete nested path: " + p, e); + } + }); } + } else { + Files.deleteIfExists(path); } - trackedFiles.clear(); } } \ No newline at end of file From 57f41ef8b52a51efc41ada44d55de6e81da9d781 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Wed, 3 Jun 2026 16:08:40 +0100 Subject: [PATCH 10/37] Enhance SIARDDKDatabaseExportModule with improved logging and graceful shutdown for asynchronous writer --- .../output/SIARDDKDatabaseExportModule.java | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index ce0fa1e01..1ae908684 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -214,14 +214,24 @@ private void startAsyncWriter() { this.writerTask = writerExecutor.submit(() -> { try { while (!Thread.currentThread().isInterrupted()) { + logger.debug("Consumer thread is waiting to take the oldest row from the queue..."); Future future = pendingRowsQueue.take(); // Enforces strict sequential order + + logger.debug("Oldest row taken. Awaiting its Virtual Thread completion (LOB HTTP boundary)..."); ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary + if (context == null) { + logger.info("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); + break; + } + super.handleDataRow(context.row()); // Alleviate disk pressure by wiping extracted structures instantly after XML // writing cleanupTransientPaths(context.transientPaths()); + logger.debug("<< DEQUEUED: Row [{}] removed from queue and successfully written. Current size: {}/{}", + context.row().getIndex(), pendingRowsQueue.size(), MAX_QUEUE_SIZE); } } catch (InterruptedException e) { Thread.currentThread().interrupt(); @@ -241,31 +251,31 @@ private void startAsyncWriter() { * failures. */ private void stopAsyncWriter() throws ModuleException { - while (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { - if (writerError.get() != null) { - break; - } + if (writerError.get() == null) { try { - TimeUnit.MILLISECONDS.sleep(20); - } catch (InterruptedException e) { + logger.debug("|| TABLE END: Injecting Poison Pill into the queue and waiting for consumer to finish..."); + + Future poisonPill = executorService.submit(() -> null); + + while (!pendingRowsQueue.offer(poisonPill, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) + break; + } + + if (writerTask != null) { + writerTask.get(); + } + + } catch (InterruptedException | ExecutionException e) { Thread.currentThread().interrupt(); + throw new ModuleException().withMessage("Failed to cleanly stop the background writer").withCause(e); } } - if (writerTask != null) { - writerTask.cancel(true); - } - - // Purge and cancel any remaining futures to avoid thread and resource leakage - if (pendingRowsQueue != null) { + // Purge any remaining futures in case of a catastrophic error + if (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { for (Future future : pendingRowsQueue) { future.cancel(true); - try { - if (future.isDone() && !future.isCancelled()) { - cleanupTransientPaths(future.get().transientPaths()); - } - } catch (Exception ignored) { - } } pendingRowsQueue.clear(); } @@ -290,12 +300,21 @@ private void enqueueRow(Row row) throws ModuleException { try { // Prevents deadlocks if the consumer thread crashes while queue is maxed out + boolean waitingLogged = false; while (!pendingRowsQueue.offer(future, 500, TimeUnit.MILLISECONDS)) { if (writerError.get() != null) { future.cancel(true); throw new ModuleException().withMessage("Pipeline halted while enqueuing row").withCause(writerError.get()); } + + if (!waitingLogged) { + logger.debug("|| PAUSED: Queue is full. Suspended database reading. Waiting for space to enqueue row [{}]...", + row.getIndex()); + waitingLogged = true; + } } + logger.debug(">> ENQUEUED: Row [{}] entered the queue. Current size: {}/{}", row.getIndex(), + pendingRowsQueue.size(), MAX_QUEUE_SIZE); } catch (InterruptedException e) { Thread.currentThread().interrupt(); future.cancel(true); @@ -329,6 +348,11 @@ private ProcessedRowContext processRowAsync(Row row) throws Exception { } } row.setCells(cells); + + long readyCount = pendingRowsQueue.stream().filter(Future::isDone).count(); + logger.debug("== READY: Row [{}] finished conversion. Currently {} ready rows waiting in queue.", row.getIndex(), + readyCount + 1); + return new ProcessedRowContext(row, transientPaths); } From 92eb31fbb26508cfaf68df1b3dff9431bbe33369 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Tue, 9 Jun 2026 15:30:42 +0100 Subject: [PATCH 11/37] Refactor LOB conversion service to support multiple converted files and enhance job status handling --- .../modules/siard/SIARDDKModuleFactory.java | 78 ++++++++++-- .../output/SIARDDKDatabaseExportModule.java | 75 ++++++++---- .../conversion/HttpLobConversionService.java | 111 ++++++++++++------ .../conversion/LobConversionService.java | 3 +- .../{ => model}/ConversionResult.java | 5 +- .../services/conversion/model/JobStatus.java | 8 ++ .../{ => model}/JobStatusResponse.java | 4 +- .../{ => model}/JobSubmissionResponse.java | 2 +- 8 files changed, 212 insertions(+), 74 deletions(-) rename dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/{ => model}/ConversionResult.java (57%) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java rename dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/{ => model}/JobStatusResponse.java (65%) rename dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/{ => model}/JobSubmissionResponse.java (94%) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java index f1bc0850a..f49f72f93 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java @@ -42,6 +42,9 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { public static final String PARAMETER_AS_SCHEMA = "as-schema"; public static final String PARAMETER_LOBS_PER_FOLDER = "lobs-per-folder"; public static final String PARAMETER_LOBS_FOLDER_SIZE = "lobs-folder-size"; + public static final String PARAMETER_LOB_CONVERSION_ENABLED = "lob-conversion"; + public static final String PARAMETER_LOB_CONVERSION_ENDPOINT = "lob-conversion-endpoint"; + public static final String PARAMETER_LOB_CONVERSION_TARGET_FORMAT = "lob-conversion-target-format"; // TODO: As things are now, are we not always generating the '.1' version of // the archive (indicating that the last .[1-9][0-9] should perhaps not be @@ -75,6 +78,20 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { .description("The maximum size (in megabytes) of the docCollection folders (default is 1000 MB").required(false) .hasArgument(true).setOptionalArgument(false).valueIfNotSet("1000"); + private static final Parameter lobConversionEnabled = new Parameter().shortName("lc") + .longName(PARAMETER_LOB_CONVERSION_ENABLED).description("Enables asynchronous LOB conversion via HTTP.") + .hasArgument(false).setOptionalArgument(false).required(false).valueIfSet("true").valueIfNotSet("false"); + + private static final Parameter lobConversionEndpoint = new Parameter().shortName("lce") + .longName(PARAMETER_LOB_CONVERSION_ENDPOINT) + .description("The API endpoint URL for the LOB conversion service (default is http://localhost:8087).") + .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet("http://localhost:8087"); + + private static final Parameter lobConversionTargetFormat = new Parameter().shortName("lcf") + .longName(PARAMETER_LOB_CONVERSION_TARGET_FORMAT) + .description("Target MIME type format for the LOB conversion (default is image/tiff).").hasArgument(true) + .setOptionalArgument(false).required(false).valueIfNotSet("image/tiff"); + // This is not used now, but will be used later // private static final Parameter clobType = new // Parameter().shortName("ct").longName("clobtype") @@ -120,6 +137,9 @@ public Map getAllParameters() { parameterMap.put(importAsSchema.longName(), importAsSchema); parameterMap.put(lobsPerFolder.longName(), lobsPerFolder); parameterMap.put(lobsFolderSize.longName(), lobsFolderSize); + parameterMap.put(lobConversionEnabled.longName(), lobConversionEnabled); + parameterMap.put(lobConversionEndpoint.longName(), lobConversionEndpoint); + parameterMap.put(lobConversionTargetFormat.longName(), lobConversionTargetFormat); // to be used later... // parameterMap.put(clobType.longName(), clobType); // parameterMap.put(clobLength.longName(), clobLength); @@ -145,18 +165,22 @@ public Parameters getExportModuleParameters() throws UnsupportedModuleException // contextDocumentationIndex, contextDocmentationFolder, // clobType, clobLength), null); - return new Parameters( - Arrays.asList( - folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) - .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), + return new Parameters(Arrays.asList( + folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) + .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEnabled.inputType(Parameter.INPUT_TYPE.COMBOBOX).possibleValues("true", "false") + .defaultSelectedIndex(1).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionTargetFormat.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), null); } @@ -193,6 +217,21 @@ public DatabaseFilterModule buildExportModule(Map parameters, pLobsFolderSize = parameters.get(lobsFolderSize); } + String pLobConversionEnabled = lobConversionEnabled.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEnabled))) { + pLobConversionEnabled = parameters.get(lobConversionEnabled); + } + + String pLobConversionEndpoint = lobConversionEndpoint.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEndpoint))) { + pLobConversionEndpoint = parameters.get(lobConversionEndpoint); + } + + String pLobConversionTargetFormat = lobConversionTargetFormat.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionTargetFormat))) { + pLobConversionTargetFormat = parameters.get(lobConversionTargetFormat); + } + // to be used later... // String pClobType = parameters.get(clobType); // String pClobLength = parameters.get(clobLength); @@ -205,6 +244,9 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleArgs.put(contextDocumentationFolder.longName(), pContextDocumentationFolder); exportModuleArgs.put(lobsPerFolder.longName(), pLobsPerFolder); exportModuleArgs.put(lobsFolderSize.longName(), pLobsFolderSize); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENABLED, pLobConversionEnabled); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENDPOINT, pLobConversionEndpoint); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_TARGET_FORMAT, pLobConversionTargetFormat); // to be used later... // exportModuleArgs.put(clobType.longName(), pClobType); @@ -227,6 +269,18 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleParameters.add(lobsFolderSize.longName()); exportModuleParameters.add(pLobsFolderSize); } + if (!pLobConversionEnabled.equals(lobConversionEnabled.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENABLED); + exportModuleParameters.add(pLobConversionEnabled); + } + if (!pLobConversionEndpoint.equals(lobConversionEndpoint.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENDPOINT); + exportModuleParameters.add(pLobConversionEndpoint); + } + if (!pLobConversionTargetFormat.equals(lobConversionTargetFormat.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_TARGET_FORMAT); + exportModuleParameters.add(pLobConversionTargetFormat); + } reporter.exportModuleParameters(getModuleName(), exportModuleParameters.toArray(new String[0])); return createSIARDDKExportModuleInstance(exportModuleArgs).getDatabaseExportModule(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 1ae908684..08461e8a7 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -36,15 +36,17 @@ import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.Row; import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.SIARDDKModuleFactory; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; -import com.databasepreservation.modules.siard.services.conversion.ConversionResult; import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; import com.databasepreservation.modules.siard.services.conversion.LobConversionService; import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.utils.ConfigUtils; /** * Handles database export pipeline matching SIARD-DK compliance, incorporating @@ -61,7 +63,8 @@ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { private ExecutorService executorService; private TempFileTracker tempFileTracker; private LobConversionService conversionService; - private static final int MAX_QUEUE_SIZE = 100; + private String targetLobFormat; + private static final Integer MAX_QUEUE_SIZE = ConfigUtils.getProperty(100, "dbptk.siarddk.export.maxQueueSize"); // Resilient Pipeline architecture attributes private BlockingQueue> pendingRowsQueue; @@ -86,10 +89,26 @@ public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { public void initDatabase() throws ModuleException { super.initDatabase(); - this.tempFileTracker = new TempFileTracker(); - String apiEndpoint = "http://localhost:8087"; - String targetFormat = "image/tiff"; - this.conversionService = new HttpLobConversionService(apiEndpoint, targetFormat, this.tempFileTracker); + Map exportModuleArgs = siarddkExportModule.getExportModuleArgs(); + boolean isLobConversionEnabled = Boolean + .parseBoolean(exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENABLED, "false")); + + if (isLobConversionEnabled) { + this.tempFileTracker = new TempFileTracker(); + String apiEndpoint = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENDPOINT, + "http://localhost:8087"); + this.targetLobFormat = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_TARGET_FORMAT, + "image/tiff"); + + this.conversionService = new HttpLobConversionService(apiEndpoint, this.targetLobFormat, this.tempFileTracker); + logger.info("LOB conversion service enabled. Endpoint: '{}', Target Format: '{}'", apiEndpoint, + this.targetLobFormat); + } else { + this.conversionService = null; + this.tempFileTracker = null; + logger.info("LOB conversion service is disabled."); + } + this.executorService = Executors.newVirtualThreadPerTaskExecutor(); this.writerExecutor = Executors.newSingleThreadExecutor(); // Dedicated single-thread pipeline consumer @@ -221,7 +240,7 @@ private void startAsyncWriter() { ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary if (context == null) { - logger.info("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); + logger.debug("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); break; } @@ -326,24 +345,36 @@ private void enqueueRow(Row row) throws ModuleException { * Processes row columns concurrently on Virtual Threads mapping extracted files * for lifecycle control. */ - private ProcessedRowContext processRowAsync(Row row) throws Exception { + private ProcessedRowContext processRowAsync(Row row) throws ModuleException { List cells = row.getCells(); List transientPaths = new ArrayList<>(); - for (int i = 0; i < cells.size(); i++) { - Cell cell = cells.get(i); - - if (cell instanceof BinaryCell binCell) { - try (InputStream originalStream = binCell.createInputStream()) { - ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); - Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFile()), - "image/tiff"); - cells.set(i, newCell); - - // Track extracted parts to clean them individually later - transientPaths.add(result.convertedFile()); - transientPaths.add(result.reportFile()); - transientPaths.add(result.convertedFile().getParent()); // directory container + if (this.conversionService != null) { + for (int i = 0; i < cells.size(); i++) { + Cell cell = cells.get(i); + + if (cell instanceof BinaryCell binCell) { + try (InputStream originalStream = binCell.createInputStream()) { + ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); + + // TODO: Handle multiple files per cell if needed. Currently assumes single file + // output. + Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFiles().getFirst()), + this.targetLobFormat); + cells.set(i, newCell); + + // Track extracted parts to clean them individually later + transientPaths.addAll(result.convertedFiles()); + transientPaths.add(result.reportFile()); + transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + } catch (Exception e) { + String errorMsg = String.format( + "Conversion failed for cell '%s' in row %d. " + + "Please check if the LOB service is running and accessible. Detail: %s", + cell.getId(), row.getIndex(), e.getMessage()); + logger.error(errorMsg); + throw new ModuleException().withMessage(errorMsg).withCause(e); + } } } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 4f143fb87..796b28398 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -15,8 +15,8 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; -import java.util.Arrays; -import java.util.Collections; +import java.util.ArrayList; +import java.util.List; import java.util.Random; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -24,6 +24,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatusResponse; +import com.databasepreservation.modules.siard.services.conversion.model.JobSubmissionResponse; +import com.databasepreservation.utils.ConfigUtils; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -38,9 +43,16 @@ public class HttpLobConversionService implements LobConversionService { private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); - private static final int MAX_NETWORK_RETRIES = 3; - private static final int BASE_POLLING_INTERVAL_MS = 2000; - private static final int MAX_POLLING_ATTEMPTS = 600; // ~20 minutes maximum wait per file before assuming Zombie Job + private static final int MAX_NETWORK_RETRIES = ConfigUtils.getProperty(3, + "dbptk.service.lob.conversion.networkRetries"); + private static final int BASE_POLLING_INTERVAL_MS = ConfigUtils.getProperty(2000, + "dbptk.service.lob.conversion.basePollingIntervalMs"); + private static final int MAX_POLLING_ATTEMPTS = ConfigUtils.getProperty(600, + "dbptk.service.lob.conversion.maxPollingAttempts"); + private static final int CONNECTION_TIMEOUT_SECONDS = ConfigUtils.getProperty(60, + "dbptk.service.lob.conversion.connectionTimeoutSeconds"); + + private static final String CRLF = "\r\n"; private final HttpClient httpClient; private final String baseUrl; @@ -54,7 +66,7 @@ public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTra this.targetFormat = targetFormat; this.fileTracker = fileTracker; this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(60)).build(); + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(CONNECTION_TIMEOUT_SECONDS)).build(); } @Override @@ -68,24 +80,20 @@ public ConversionResult convertLob(String cellId, InputStream inputStream) throw private String submitJob(String cellId, InputStream inputStream) throws Exception { String boundary = "DbptkBoundary" + System.currentTimeMillis(); - String header = "--" + boundary + "\r\n" + "Content-Disposition: form-data; name=\"targetFormat\"\r\n\r\n" - + targetFormat + "\r\n" + "--" + boundary + "\r\n" - + "Content-Disposition: form-data; name=\"file\"; filename=\"lob_" + cellId + ".bin\"\r\n" - + "Content-Type: application/octet-stream\r\n\r\n"; - - String footer = "\r\n--" + boundary + "--\r\n"; + String header = buildMultipartHeader(boundary, cellId); + String footer = CRLF + "--" + boundary + "--" + CRLF; InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); - InputStream multipartStream = new SequenceInputStream( - Collections.enumeration(Arrays.asList(headerStream, inputStream, footerStream))); + InputStream multipartStream = new SequenceInputStream(new SequenceInputStream(headerStream, inputStream), + footerStream); HttpRequest submitRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs")) .header("Content-Type", "multipart/form-data; boundary=" + boundary) .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); - HttpResponse submitResponse = httpClient.send(submitRequest, BodyHandlers.ofString()); + HttpResponse submitResponse = executeWithRetry(submitRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); if (submitResponse.statusCode() >= 400) { log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), @@ -106,21 +114,23 @@ private void waitForCompletion(String cellId, String jobId) throws Exception { HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); - JobStatusResponse status = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); + JobStatusResponse response = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); - switch (status.status().toUpperCase()) { - case "COMPLETED", "DONE", "SUCCESS" -> { + switch (response.status()) { + case JobStatus.COMPLETED -> { log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); return; } - case "FAILED", "ERROR", "EVICTED" -> { - log.error("API reported terminal failure for Job {} (Cell {})", jobId, cellId); - throw new RuntimeException("Server failed to convert cell: " + cellId); + case JobStatus.FAILED, JobStatus.EVICTED -> { + log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, + response.status()); + throw new RuntimeException( + "Server failed to convert cell: " + cellId + " (Status: " + response.status() + ")"); } - default -> { + case JobStatus.ACCEPTED, JobStatus.PROCESSING -> { if (attempts % 30 == 0) { log.warn("Job {} (Cell {}) is taking unusually long. Current status: {}. Attempt: {}/{}", jobId, cellId, - status.status(), attempts, MAX_POLLING_ATTEMPTS); + response.status(), attempts, MAX_POLLING_ATTEMPTS); } long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); Thread.sleep(sleepTime); @@ -157,35 +167,37 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); - Path convertedLob = null; + Path normalizedExtractionDir = extractionDir.normalize(); + + List convertedFiles = new ArrayList<>(); Path reportFile = null; try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tempZipFile.toFile()))) { ZipEntry zipEntry; while ((zipEntry = zis.getNextEntry()) != null) { - Path extractedFilePath = extractionDir.resolve(zipEntry.getName()); + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); - if (!extractedFilePath.normalize().startsWith(extractionDir)) { - throw new SecurityException("Corrupted ZIP entry: " + zipEntry.getName()); + if (!extractedFilePath.startsWith(normalizedExtractionDir)) { + throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); } if (!zipEntry.isDirectory()) { - Files.copy(zis, extractedFilePath); + Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); if (zipEntry.getName().toLowerCase().contains("report")) { reportFile = extractedFilePath; } else { - convertedLob = extractedFilePath; + convertedFiles.add(extractedFilePath); } } } } - if (convertedLob == null || reportFile == null) { - throw new RuntimeException("Downloaded ZIP lacks expected format (LOB + Report) for cell: " + cellId); + if (convertedFiles.isEmpty() || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); } - return new ConversionResult(convertedLob, reportFile); + return new ConversionResult(convertedFiles, reportFile); } private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, @@ -194,16 +206,47 @@ private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.B for (int attempt = 1; attempt <= maxRetries; attempt++) { try { - return httpClient.send(request, responseBodyHandler); + HttpResponse response = httpClient.send(request, responseBodyHandler); + + if (response.statusCode() >= 500) { + throw new IOException("Temporary server error: " + response.statusCode() + " - " + response.body()); + } + + return response; } catch (IOException e) { lastException = e; log.warn("Attempt {} failed for {}: {}", attempt, request.uri(), e.getMessage()); if (attempt == maxRetries) break; - Thread.sleep((long) Math.pow(2, attempt) * 1000); + + try { + Thread.sleep((long) Math.pow(2, attempt) * 1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted for URI: " + request.uri(), ie); + } } } throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); } + + private String buildMultipartHeader(String boundary, String cellId) { + StringBuilder sb = new StringBuilder(); + + // Target format part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"targetFormat\"").append(CRLF); + sb.append(CRLF); + sb.append(this.targetFormat).append(CRLF); + + // File part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"file\"; filename=\"lob_").append(cellId).append(".bin\"") + .append(CRLF); + sb.append("Content-Type: application/octet-stream").append(CRLF); + sb.append(CRLF); + + return sb.toString(); + } } \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java index 038e779b4..8c7d8622a 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -2,7 +2,8 @@ import java.io.InputStream; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; + public interface LobConversionService { ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception; } - diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java similarity index 57% rename from dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java rename to dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java index 4b48a9bc5..1071de86f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/ConversionResult.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java @@ -1,9 +1,10 @@ -package com.databasepreservation.modules.siard.services.conversion; +package com.databasepreservation.modules.siard.services.conversion.model; import java.nio.file.Path; +import java.util.List; /** * @author Gabriel Barros */ -public record ConversionResult(Path convertedFile, Path reportFile) { +public record ConversionResult(List convertedFiles, Path reportFile) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java new file mode 100644 index 000000000..5e7107946 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public enum JobStatus { + ACCEPTED, PROCESSING, COMPLETED, FAILED, EVICTED; +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java similarity index 65% rename from dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java rename to dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java index 79f8551c3..4028cd94f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobStatusResponse.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java @@ -1,7 +1,7 @@ -package com.databasepreservation.modules.siard.services.conversion; +package com.databasepreservation.modules.siard.services.conversion.model; /** * @author Gabriel Barros */ -public record JobStatusResponse(String status) { +public record JobStatusResponse(JobStatus status) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java similarity index 94% rename from dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java rename to dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java index 2abfaf6dd..933301871 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/JobSubmissionResponse.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java @@ -1,4 +1,4 @@ -package com.databasepreservation.modules.siard.services.conversion; +package com.databasepreservation.modules.siard.services.conversion.model; /** * @author Gabriel Barros From 3a2de873eae42877e7fd8088bb806cf765574845 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Mon, 15 Jun 2026 09:15:41 +0100 Subject: [PATCH 12/37] Add conversion service options to parameter categories and update input types --- .../databasepreservation/model/parameters/Parameter.java | 6 ++++-- .../modules/siard/SIARDDKModuleFactory.java | 8 ++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java index 03f74e006..378250821 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java @@ -28,7 +28,7 @@ public enum INPUT_TYPE { /* GUI Helper for SIARD Export Module */ public enum CATEGORY_TYPE { - SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, NONE + SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, CONVERSION_SERVICE_OPTIONS, NONE } public enum FILE_FILTER_TYPE { @@ -312,7 +312,9 @@ public Parameter defaultSelectedIndex(Integer index) { return this; } - public Integer getDefaultSelectedIndex() { return defaultSelectedIndex; } + public Integer getDefaultSelectedIndex() { + return defaultSelectedIndex; + } /** * Gets the export option type for this parameter; Helper to automatize the diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java index f49f72f93..87e038dcd 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java @@ -176,11 +176,11 @@ public Parameters getExportModuleParameters() throws UnsupportedModuleException .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobConversionEnabled.inputType(Parameter.INPUT_TYPE.COMBOBOX).possibleValues("true", "false") - .defaultSelectedIndex(1).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEnabled.inputType(Parameter.INPUT_TYPE.CHECKBOX).exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), + lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), lobConversionTargetFormat.inputType(Parameter.INPUT_TYPE.TEXT) - .exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS)), null); } From 792961ae0514b8e030d9330b83df4c812261490e Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Tue, 16 Jun 2026 17:11:33 +0100 Subject: [PATCH 13/37] Adapt integration with lob conversion plugin to properly support binaries in siarddk --- .../content/SIARDDKContentExportStrategy.java | 123 ++++++++++-------- .../output/SIARDDKDatabaseExportModule.java | 4 +- .../SIARDDKContentPathExportStrategy.java | 11 +- .../conversion/HttpLobConversionService.java | 58 ++++++++- .../conversion/model/ConversionResult.java | 2 +- 5 files changed, 135 insertions(+), 63 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 9bb53fe9d..02fb84020 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,7 +7,6 @@ */ package com.databasepreservation.modules.siard.out.content; -import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; @@ -15,6 +14,9 @@ import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.codec.binary.Hex; import org.apache.commons.io.IOUtils; @@ -36,7 +38,6 @@ import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.SchemaStructure; import com.databasepreservation.model.structure.TableStructure; -import com.databasepreservation.modules.siard.common.LargeObject; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDConstants; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; @@ -45,6 +46,7 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.fasterxml.jackson.databind.ObjectMapper; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -320,60 +322,77 @@ public Row tableRow(Row row) throws ModuleException { final BinaryCell binaryCell = (BinaryCell) cell; - // BLOB is not NULL - - double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - lobsTracker.addLOB(lobSizeMB); // Only if LOB not NULL - - // Determine the mimetype (Tika should use an inputstream which - // supports marks) - - InputStream is = new BufferedInputStream(binaryCell.createInputStream()); - // Removed because TIKA was a security vulnerability and this feature was not - // needed/not fully implemented (see #341) + double lobSizeTotal = 0; String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; - IOUtils.closeQuietly(is); - - // Archive BLOB - simultaneous writing always supported for - // SIARDDK - - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - - String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); - String fileExtension; - if (mimetypeHandler.isMimetypeAllowed(mimeType)) { - fileExtension = mimetypeHandler.getFileExtension(mimeType); - } else { - fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; - // Log (table level) that unknown BLOB mimetype was detected - foundUnknownMimetype = true; - } - path += fileExtension; - LargeObject blob = new LargeObject(binaryCell, path); - - // Create new FileIndexFileStrategy - - // Write the BLOB - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), - writeStrategy); - InputStream in = blob.getInputStreamProvider().createInputStream(); - IOUtils.copy(in, out); - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(out); - blob.getInputStreamProvider().cleanResources(); - - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) + if (mimeType.equals("application/zip")) { + // First pass to read report json + Map report = null; + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) { + ObjectMapper mapper = new ObjectMapper(); + report = mapper.readValue(zis.readAllBytes(), Map.class); + } + } + } + if (report == null) { + throw new ModuleException().withMessage( + "Could not find report in zip file for BLOB in table " + tableCounter + ", column " + columnIndex); + } + + List> processedArtifacts = ((List>) report + .get("processedArtifacts")); + + String originalFileName = processedArtifacts.getFirst().get("originalName"); + String firstProcessedFileMimeType = processedArtifacts.getFirst().get("finalFormat"); + + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + int fileCount = 0; + while ((zipEntry = zis.getNextEntry()) != null) { + // Archive BLOB - simultaneous writing always supported for + // SIARDDK + if (!zipEntry.getName().toLowerCase().contains("report")) { + fileCount++; + lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); + Map processedArtifactReport = null; + for (Map artifact : processedArtifacts) { + if (artifact.get("finalFileName").equals(zipEntry.getName())) { + processedArtifactReport = artifact; + break; + } + } + String fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); + outputPath += fileCount + "." + fileExtension; + + // Write the BLOB + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, + writeStrategy); + zis.transferTo(out); + + // Add file to fileIndex + SIARDDKFileIndexFileStrategy.addFile(outputPath); + } + } + } catch (IOException e) { + throw new ModuleException(); + } + lobsTracker.addLOB(lobSizeTotal); // Only if LOB not NULL + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - // TO-DO: obtain (how?) hardcoded values - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - "originalFilename", fileExtension, null); + // Add file to docIndex (a lot easier to do here even though we + // are dealing with metadata) - // Add file to fileIndex - SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + // TO-DO: obtain (how?) hardcoded values + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFileName, mimetypeHandler.getFileExtension(firstProcessedFileMimeType), null); + } } else { // never happens diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 08461e8a7..42d208a9e 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -359,8 +359,8 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { // TODO: Handle multiple files per cell if needed. Currently assumes single file // output. - Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.convertedFiles().getFirst()), - this.targetLobFormat); + Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), + "application/zip"); cells.set(i, newCell); // Track extracted parts to clean them individually later diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java index dcbb3fce0..98731d8f1 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java @@ -7,10 +7,11 @@ */ package com.databasepreservation.modules.siard.out.path; +import org.apache.commons.lang3.NotImplementedException; + import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; -import org.apache.commons.lang3.NotImplementedException; /** * @author Andreas Kring @@ -24,14 +25,13 @@ public class SIARDDKContentPathExportStrategy implements ContentPathExportStrate private static final String SCHEMA_DIR = "schema"; private static final String DOCUMENT_DIR = "Documents"; private static final String DOC_COLLECTION = "docCollection"; - private static final String fileCount = "1"; // Design decision private LOBsTracker lobsTracker; public SIARDDKContentPathExportStrategy(LOBsTracker loBsTracker) { this.lobsTracker = loBsTracker; } - + public SIARDDKContentPathExportStrategy(SIARDDKExportModule siarddkExportModule) { this.lobsTracker = siarddkExportModule.getLobsTracker(); } @@ -71,13 +71,12 @@ public String getBlobFilePath(int schemaIndex, int tableIndex, int columnIndex, // TO-DO: add test case int docCollectionCount = lobsTracker.getDocCollectionCount(); - int LOBsCount = lobsTracker.getLOBsCount(); + int LOBsCount = lobsTracker.getLOBsCount() + 1; // Note: code assumes one file in each folder return new StringBuilder().append(DOCUMENT_DIR).append(SIARDDKConstants.FILE_SEPARATOR).append(DOC_COLLECTION) .append(docCollectionCount).append(SIARDDKConstants.FILE_SEPARATOR).append(LOBsCount) - .append(SIARDDKConstants.FILE_SEPARATOR).append(fileCount).append(SIARDDKConstants.FILE_EXTENSION_SEPARATOR) - .toString(); + .append(SIARDDKConstants.FILE_SEPARATOR).toString(); } // Not used in SIARDDK diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 796b28398..5c8ffc780 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -74,7 +74,7 @@ public ConversionResult convertLob(String cellId, InputStream inputStream) throw log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); - return downloadAndExtractResult(cellId, jobId); + return downloadResult(cellId, jobId); } private String submitJob(String cellId, InputStream inputStream) throws Exception { @@ -163,6 +163,23 @@ private ConversionResult downloadAndExtractResult(String cellId, String jobId) t } } + /** + * Downloads the resulting ZIP and lists its contents, returning the compressed + * file. + */ + private ConversionResult downloadResult(String cellId, String jobId) throws Exception { + Path zipFile = Files.createFile(Path.of("siarddk_conv_" + cellId + "_" + jobId + ".zip")); + // TODO: Should this still be tracked? + fileTracker.track(zipFile); + + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); + + executeWithRetry(downloadRequest, BodyHandlers.ofFile(zipFile), MAX_NETWORK_RETRIES); + + return listZipContents(cellId, zipFile); + } + private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); @@ -197,7 +214,44 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); } - return new ConversionResult(convertedFiles, reportFile); + return new ConversionResult(convertedFiles, reportFile, tempZipFile); + } + + private ConversionResult listZipContents(String cellId, Path zipFile) throws Exception { + Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); + fileTracker.trackDir(extractionDir); + + Path normalizedExtractionDir = extractionDir.normalize(); + + List convertedFiles = new ArrayList<>(); + Path reportFile = null; + + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile.toFile()))) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); + + if (!extractedFilePath.startsWith(normalizedExtractionDir)) { + throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); + } + + if (!zipEntry.isDirectory()) { + Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + + if (zipEntry.getName().toLowerCase().contains("report")) { + reportFile = extractedFilePath; + } else { + convertedFiles.add(extractedFilePath); + } + } + } + } + + if (convertedFiles.isEmpty() || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); + } + + return new ConversionResult(convertedFiles, reportFile, zipFile); } private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java index 1071de86f..5d229bb2d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java @@ -6,5 +6,5 @@ /** * @author Gabriel Barros */ -public record ConversionResult(List convertedFiles, Path reportFile) { +public record ConversionResult(List convertedFiles, Path reportFile, Path zipFile) { } From e8c2862e1b9211aeb8116e6fbba09f3fde4ac63d Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Wed, 17 Jun 2026 15:43:00 +0100 Subject: [PATCH 14/37] Ignore invalid mimetypes and unprocessed files in SIARDDK export --- .../content/SIARDDKContentExportStrategy.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 02fb84020..89d182d42 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -13,6 +13,7 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.security.InvalidParameterException; import java.util.List; import java.util.Map; import java.util.zip.ZipEntry; @@ -354,11 +355,11 @@ public Row tableRow(Row row) throws ModuleException { while ((zipEntry = zis.getNextEntry()) != null) { // Archive BLOB - simultaneous writing always supported for // SIARDDK + + // Skip report file if (!zipEntry.getName().toLowerCase().contains("report")) { - fileCount++; - lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); - String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); + // Find processing report for current entry Map processedArtifactReport = null; for (Map artifact : processedArtifacts) { if (artifact.get("finalFileName").equals(zipEntry.getName())) { @@ -366,10 +367,31 @@ public Row tableRow(Row row) throws ModuleException { break; } } - String fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); - outputPath += fileCount + "." + fileExtension; + if (processedArtifactReport == null) { + logger.warn( + "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", + zipEntry.getName(), tableCounter, columnIndex); + break; + } + + // Get processed file extension + String fileExtension = null; + try { + fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); + } catch (InvalidParameterException e) { + logger.warn( + "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", + zipEntry.getName(), tableCounter, columnIndex); + break; + } + + // Increment file trackings + fileCount++; + lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); // Write the BLOB + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); + outputPath += fileCount + "." + fileExtension; OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, writeStrategy); zis.transferTo(out); From 108852085e93792d4f70dba64826382748b36074 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Wed, 17 Jun 2026 16:45:52 +0100 Subject: [PATCH 15/37] Default SIARDDK mimetype and processing status check --- .../out/content/SIARDDKContentExportStrategy.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 89d182d42..620bf3131 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -347,7 +347,8 @@ public Row tableRow(Row row) throws ModuleException { .get("processedArtifacts")); String originalFileName = processedArtifacts.getFirst().get("originalName"); - String firstProcessedFileMimeType = processedArtifacts.getFirst().get("finalFormat"); + // Default to tiff, attempt to find real mimetype as we go through zip entries + String processedFilesExtension = "tif"; try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; @@ -367,7 +368,7 @@ public Row tableRow(Row row) throws ModuleException { break; } } - if (processedArtifactReport == null) { + if (processedArtifactReport == null || !processedArtifactReport.get("status").equals("CONVERTED")) { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", zipEntry.getName(), tableCounter, columnIndex); @@ -385,6 +386,9 @@ public Row tableRow(Row row) throws ModuleException { break; } + // Set overall document mimetype + processedFilesExtension = fileExtension; + // Increment file trackings fileCount++; lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); @@ -413,7 +417,7 @@ public Row tableRow(Row row) throws ModuleException { // TO-DO: obtain (how?) hardcoded values SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFileName, mimetypeHandler.getFileExtension(firstProcessedFileMimeType), null); + originalFileName, mimetypeHandler.getFileExtension(processedFilesExtension), null); } } else { From 537b0ea7a26521de666c48f89518d556e244b8e6 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Thu, 18 Jun 2026 10:16:32 +0100 Subject: [PATCH 16/37] Don't convert extension twice in SIARDDKContentExportStrategy --- .../content/SIARDDKContentExportStrategy.java | 43 +++++++++---------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 620bf3131..78ffc35b7 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,27 +7,6 @@ */ package com.databasepreservation.modules.siard.out.content; -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.security.InvalidParameterException; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; - -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.jdom2.Document; -import org.jdom2.Element; -import org.jdom2.Namespace; -import org.jdom2.output.XMLOutputter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.ComposedCell; @@ -48,6 +27,26 @@ import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; import com.fasterxml.jackson.databind.ObjectMapper; +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.IOUtils; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.Namespace; +import org.jdom2.output.XMLOutputter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.security.InvalidParameterException; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -417,7 +416,7 @@ public Row tableRow(Row row) throws ModuleException { // TO-DO: obtain (how?) hardcoded values SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFileName, mimetypeHandler.getFileExtension(processedFilesExtension), null); + originalFileName, processedFilesExtension, null); } } else { From 366bc1c354666875f83d71f5c0558865224f3ae8 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Thu, 18 Jun 2026 10:23:36 +0100 Subject: [PATCH 17/37] Only write column data for lob columns if lobs are successfully processed --- .../content/SIARDDKContentExportStrategy.java | 70 ++++++++++--------- 1 file changed, 38 insertions(+), 32 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 78ffc35b7..e623e3a46 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,6 +7,27 @@ */ package com.databasepreservation.modules.siard.out.content; +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.OutputStreamWriter; +import java.io.UnsupportedEncodingException; +import java.security.InvalidParameterException; +import java.util.List; +import java.util.Map; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.IOUtils; +import org.jdom2.Document; +import org.jdom2.Element; +import org.jdom2.Namespace; +import org.jdom2.output.XMLOutputter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.ComposedCell; @@ -27,26 +48,6 @@ import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.jdom2.Document; -import org.jdom2.Element; -import org.jdom2.Namespace; -import org.jdom2.output.XMLOutputter; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedWriter; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.OutputStreamWriter; -import java.io.UnsupportedEncodingException; -import java.security.InvalidParameterException; -import java.util.List; -import java.util.Map; -import java.util.zip.ZipEntry; -import java.util.zip.ZipInputStream; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -349,9 +350,10 @@ public Row tableRow(Row row) throws ModuleException { // Default to tiff, attempt to find real mimetype as we go through zip entries String processedFilesExtension = "tif"; + int fileCount = 0; try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; - int fileCount = 0; + while ((zipEntry = zis.getNextEntry()) != null) { // Archive BLOB - simultaneous writing always supported for // SIARDDK @@ -371,7 +373,7 @@ public Row tableRow(Row row) throws ModuleException { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", zipEntry.getName(), tableCounter, columnIndex); - break; + continue; } // Get processed file extension @@ -382,7 +384,7 @@ public Row tableRow(Row row) throws ModuleException { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", zipEntry.getName(), tableCounter, columnIndex); - break; + continue; } // Set overall document mimetype @@ -407,16 +409,20 @@ public Row tableRow(Row row) throws ModuleException { throw new ModuleException(); } lobsTracker.addLOB(lobSizeTotal); // Only if LOB not NULL - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) - - // TO-DO: obtain (how?) hardcoded values - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFileName, processedFilesExtension, null); + if (fileCount > 0) { + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + + // Add file to docIndex (a lot easier to do here even though we + // are dealing with metadata) + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, + lobsTracker.getDocCollectionCount(), originalFileName, processedFilesExtension, null); + } else { + tableXmlWriter.append(TAB).append(TAB).append("\n"); + } } } else { From ec37745ab9456fdd57f8bbf79c200697b81989b5 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Fri, 19 Jun 2026 14:28:21 +0100 Subject: [PATCH 18/37] Refactor HttpLobConversionService to improve exception handling and update method signatures for better error reporting --- .../content/SIARDDKContentExportStrategy.java | 19 +++++++---- .../output/SIARDDKDatabaseExportModule.java | 15 ++++++--- .../conversion/HttpLobConversionService.java | 22 ++++++++----- .../HttpLobConversionServiceException.java | 33 +++++++++++++++++++ .../conversion/LobConversionService.java | 5 ++- 5 files changed, 73 insertions(+), 21 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index e623e3a46..b810a754c 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -290,16 +290,14 @@ public Row tableRow(Row row) throws ModuleException { binaryCell.cleanResources(); } } else { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } } else { // cell must contain BLOB or CLOB if (cell instanceof NullCell) { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } else if (cell instanceof SimpleCell) { // CLOB is not NULL @@ -420,9 +418,13 @@ public Row tableRow(Row row) throws ModuleException { SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), originalFileName, processedFilesExtension, null); } else { - tableXmlWriter.append(TAB).append(TAB).append("\n"); + whiteNilCell(columnIndex); } + } else { + logger.warn( + "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", + mimeType, tableCounter, columnIndex); + whiteNilCell(columnIndex); } } else { @@ -440,6 +442,11 @@ public Row tableRow(Row row) throws ModuleException { return row; } + private void whiteNilCell(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + } + @Override public void setOnceReporter(Reporter reporter) { this.reporter = reporter; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 42d208a9e..06b2e6b5b 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -43,6 +43,7 @@ import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionServiceException; import com.databasepreservation.modules.siard.services.conversion.LobConversionService; import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; @@ -367,13 +368,17 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { transientPaths.addAll(result.convertedFiles()); transientPaths.add(result.reportFile()); transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container - } catch (Exception e) { + } catch (IOException | InterruptedException | HttpLobConversionServiceException e) { + String statusCodeInfo = ""; + if (e instanceof HttpLobConversionServiceException apiEx && apiEx.getHttpStatusCode() != null) { + statusCodeInfo = " (HTTP " + apiEx.getHttpStatusCode() + ")"; + } + String errorMsg = String.format( - "Conversion failed for cell '%s' in row %d. " - + "Please check if the LOB service is running and accessible. Detail: %s", - cell.getId(), row.getIndex(), e.getMessage()); + "Conversion failed for cell '%s' in row %d%s. " + "Pipeline continuing with original file. Detail: %s", + cell.getId(), row.getIndex(), statusCodeInfo, e.getMessage()); + logger.error(errorMsg); - throw new ModuleException().withMessage(errorMsg).withCause(e); } } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 5c8ffc780..83256da38 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -70,14 +70,16 @@ public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTra } @Override - public ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception { + public ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); return downloadResult(cellId, jobId); } - private String submitJob(String cellId, InputStream inputStream) throws Exception { + private String submitJob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { String boundary = "DbptkBoundary" + System.currentTimeMillis(); String header = buildMultipartHeader(boundary, cellId); @@ -98,7 +100,8 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio if (submitResponse.statusCode() >= 400) { log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), submitResponse.body()); - throw new RuntimeException("Failed to submit LOB for cell " + cellId); + throw new HttpLobConversionServiceException("Failed to submit LOB for cell " + cellId, + submitResponse.statusCode()); } JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); @@ -106,7 +109,8 @@ private String submitJob(String cellId, InputStream inputStream) throws Exceptio return job.id(); } - private void waitForCompletion(String cellId, String jobId) throws Exception { + private void waitForCompletion(String cellId, String jobId) + throws IOException, InterruptedException, HttpLobConversionServiceException { log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); for (int attempts = 1; attempts <= MAX_POLLING_ATTEMPTS; attempts++) { @@ -124,7 +128,7 @@ private void waitForCompletion(String cellId, String jobId) throws Exception { case JobStatus.FAILED, JobStatus.EVICTED -> { log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, response.status()); - throw new RuntimeException( + throw new HttpLobConversionServiceException( "Server failed to convert cell: " + cellId + " (Status: " + response.status() + ")"); } case JobStatus.ACCEPTED, JobStatus.PROCESSING -> { @@ -139,7 +143,7 @@ private void waitForCompletion(String cellId, String jobId) throws Exception { } log.error("Zombie Job detected. API failed to resolve Job {} (Cell {}) within the maximum polling threshold.", jobId, cellId); - throw new RuntimeException("Timeout after waiting for conversion of cell: " + cellId); + throw new HttpLobConversionServiceException("Timeout after waiting for conversion of cell: " + cellId); } /** @@ -167,7 +171,7 @@ private ConversionResult downloadAndExtractResult(String cellId, String jobId) t * Downloads the resulting ZIP and lists its contents, returning the compressed * file. */ - private ConversionResult downloadResult(String cellId, String jobId) throws Exception { + private ConversionResult downloadResult(String cellId, String jobId) throws IOException, InterruptedException { Path zipFile = Files.createFile(Path.of("siarddk_conv_" + cellId + "_" + jobId + ".zip")); // TODO: Should this still be tracked? fileTracker.track(zipFile); @@ -217,7 +221,7 @@ private ConversionResult extractZipContents(String cellId, Path tempZipFile) thr return new ConversionResult(convertedFiles, reportFile, tempZipFile); } - private ConversionResult listZipContents(String cellId, Path zipFile) throws Exception { + private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); @@ -255,7 +259,7 @@ private ConversionResult listZipContents(String cellId, Path zipFile) throws Exc } private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, - int maxRetries) throws Exception { + int maxRetries) throws InterruptedException, IOException { Exception lastException = null; for (int attempt = 1; attempt <= maxRetries; attempt++) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java new file mode 100644 index 000000000..9fc7d9960 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java @@ -0,0 +1,33 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public class HttpLobConversionServiceException extends Exception { + + private final Integer httpStatusCode; + + public HttpLobConversionServiceException(String message) { + super(message); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, Throwable cause) { + super(message, cause); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode) { + super(message); + this.httpStatusCode = httpStatusCode; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode, Throwable cause) { + super(message, cause); + this.httpStatusCode = httpStatusCode; + } + + public Integer getHttpStatusCode() { + return httpStatusCode; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java index 8c7d8622a..0f26c5b32 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -1,9 +1,12 @@ package com.databasepreservation.modules.siard.services.conversion; +import java.io.IOException; import java.io.InputStream; +import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; public interface LobConversionService { - ConversionResult convertLob(String cellId, InputStream inputStream) throws Exception; + ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, ModuleException, InterruptedException, HttpLobConversionServiceException; } From 80cc5af63c5db9b20903a8b437ddedd843ae322b Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Sun, 21 Jun 2026 11:10:41 +0100 Subject: [PATCH 19/37] Refactor SIARDDKContentExportStrategy to improve artifact handling and update file processing logic --- .../content/SIARDDKContentExportStrategy.java | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index b810a754c..9996d93da 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -341,10 +341,9 @@ public Row tableRow(Row row) throws ModuleException { "Could not find report in zip file for BLOB in table " + tableCounter + ", column " + columnIndex); } - List> processedArtifacts = ((List>) report - .get("processedArtifacts")); + List> artifacts = (List>) report.get("artifacts"); - String originalFileName = processedArtifacts.getFirst().get("originalName"); + String originalFileName = (String) report.get("originalInputFile"); // Default to tiff, attempt to find real mimetype as we go through zip entries String processedFilesExtension = "tif"; @@ -360,14 +359,16 @@ public Row tableRow(Row row) throws ModuleException { if (!zipEntry.getName().toLowerCase().contains("report")) { // Find processing report for current entry - Map processedArtifactReport = null; - for (Map artifact : processedArtifacts) { - if (artifact.get("finalFileName").equals(zipEntry.getName())) { + Map processedArtifactReport = null; + for (Map artifact : artifacts) { + if (artifact.get("logicalName").equals(zipEntry.getName())) { processedArtifactReport = artifact; break; } } - if (processedArtifactReport == null || !processedArtifactReport.get("status").equals("CONVERTED")) { + boolean isBypassed = processedArtifactReport != null ? (Boolean) processedArtifactReport.get("isBypassed") : true; + + if (processedArtifactReport == null || isBypassed) { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", zipEntry.getName(), tableCounter, columnIndex); @@ -377,7 +378,8 @@ public Row tableRow(Row row) throws ModuleException { // Get processed file extension String fileExtension = null; try { - fileExtension = mimetypeHandler.getFileExtension(processedArtifactReport.get("finalFormat")); + String finalMimeType = (String) processedArtifactReport.get("finalMimeType"); + fileExtension = mimetypeHandler.getFileExtension(finalMimeType); } catch (InvalidParameterException e) { logger.warn( "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", From 35d4e72c75d9291edb3a02dedd519942182bc193 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Sun, 21 Jun 2026 15:16:49 +0100 Subject: [PATCH 20/37] Enhance SIARDDKContentExportStrategy with LOB conversion auditing and report extraction --- .../content/SIARDDKContentExportStrategy.java | 208 +++++++++--------- .../conversion/LobConversionAuditor.java | 41 ++++ .../model/report/ArtifactReport.java | 13 ++ .../model/report/AuditTrailStep.java | 13 ++ .../model/report/ConversionReport.java | 17 ++ .../conversion/model/report/DbptkContext.java | 9 + 6 files changed, 198 insertions(+), 103 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 9996d93da..a796e4086 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -13,9 +13,9 @@ import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; -import java.security.InvalidParameterException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; -import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; @@ -47,6 +47,10 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.databasepreservation.modules.siard.services.conversion.LobConversionAuditor; +import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.DbptkContext; import com.fasterxml.jackson.databind.ObjectMapper; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -72,6 +76,9 @@ public class SIARDDKContentExportStrategy implements ContentExportStrategy { private final LOBsTracker lobsTracker; private final MimetypeHandler mimetypeHandler; + private final LobConversionAuditor auditor; + private final ObjectMapper mapper; + private Reporter reporter; public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { @@ -88,6 +95,11 @@ public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { baseContainer = siarddkExportModule.getMainContainer(); writeStrategy = siarddkExportModule.getWriteStrategy(); lobsTracker = siarddkExportModule.getLobsTracker(); + + this.mapper = new ObjectMapper(); + Path exportRoot = baseContainer.getPath().getParent(); + String archiveName = baseContainer.getPath().getFileName().toString(); + this.auditor = new LobConversionAuditor(exportRoot, archiveName); } @Override @@ -318,117 +330,20 @@ public Row tableRow(Row row) throws ModuleException { } else if (cell instanceof BinaryCell) { // BLOB case - final BinaryCell binaryCell = (BinaryCell) cell; - - double lobSizeTotal = 0; String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + // ------------------------------------------------------------- + // BLOB EXTRACTION DELEGATION + // ------------------------------------------------------------- if (mimeType.equals("application/zip")) { - // First pass to read report json - Map report = null; - try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { - ZipEntry zipEntry; - while ((zipEntry = zis.getNextEntry()) != null) { - if (zipEntry.getName().toLowerCase().contains("report")) { - ObjectMapper mapper = new ObjectMapper(); - report = mapper.readValue(zis.readAllBytes(), Map.class); - } - } - } - if (report == null) { - throw new ModuleException().withMessage( - "Could not find report in zip file for BLOB in table " + tableCounter + ", column " + columnIndex); - } - - List> artifacts = (List>) report.get("artifacts"); - - String originalFileName = (String) report.get("originalInputFile"); - // Default to tiff, attempt to find real mimetype as we go through zip entries - String processedFilesExtension = "tif"; - - int fileCount = 0; - try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { - ZipEntry zipEntry; - - while ((zipEntry = zis.getNextEntry()) != null) { - // Archive BLOB - simultaneous writing always supported for - // SIARDDK - - // Skip report file - if (!zipEntry.getName().toLowerCase().contains("report")) { - - // Find processing report for current entry - Map processedArtifactReport = null; - for (Map artifact : artifacts) { - if (artifact.get("logicalName").equals(zipEntry.getName())) { - processedArtifactReport = artifact; - break; - } - } - boolean isBypassed = processedArtifactReport != null ? (Boolean) processedArtifactReport.get("isBypassed") : true; - - if (processedArtifactReport == null || isBypassed) { - logger.warn( - "Ignoring file {} in zip file for BLOB in table {}, column {} since it has not been processed.", - zipEntry.getName(), tableCounter, columnIndex); - continue; - } - - // Get processed file extension - String fileExtension = null; - try { - String finalMimeType = (String) processedArtifactReport.get("finalMimeType"); - fileExtension = mimetypeHandler.getFileExtension(finalMimeType); - } catch (InvalidParameterException e) { - logger.warn( - "Ignoring file {} in zip file for BLOB in table {}, column {} since it has an invalid mimetype.", - zipEntry.getName(), tableCounter, columnIndex); - continue; - } - - // Set overall document mimetype - processedFilesExtension = fileExtension; - - // Increment file trackings - fileCount++; - lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); - - // Write the BLOB - String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); - outputPath += fileCount + "." + fileExtension; - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, - writeStrategy); - zis.transferTo(out); - - // Add file to fileIndex - SIARDDKFileIndexFileStrategy.addFile(outputPath); - } - } - } catch (IOException e) { - throw new ModuleException(); - } - lobsTracker.addLOB(lobSizeTotal); // Only if LOB not NULL - - if (fileCount > 0) { - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); - - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, - lobsTracker.getDocCollectionCount(), originalFileName, processedFilesExtension, null); - } else { - whiteNilCell(columnIndex); - } + processConvertedLobArchive(binaryCell, columnIndex); } else { logger.warn( "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", mimeType, tableCounter, columnIndex); whiteNilCell(columnIndex); } - } else { // never happens } @@ -444,6 +359,93 @@ public Row tableRow(Row row) throws ModuleException { return row; } + private void processConvertedLobArchive(BinaryCell binaryCell, int columnIndex) throws ModuleException { + try { + ConversionReport report = extractReportFromZip(binaryCell); + if (report == null) { + throw new ModuleException().withMessage("Missing conversion_report.json in cell archive."); + } + + List siardPhysicalPaths = new ArrayList<>(); + int fileCount = 0; + String processedFilesExtension = "tif"; + + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) + continue; + + ArtifactReport artifactMeta = findArtifactMetadata(report.artifacts(), zipEntry.getName()); + + if (artifactMeta == null || artifactMeta.isBypassed()) { + logger.warn("Ignoring bypassed or unknown file: {}. Reason: {}", zipEntry.getName(), + artifactMeta != null ? artifactMeta.errorMessage() : "Not in report"); + continue; + } + + String fileExt = mimetypeHandler.getFileExtension(artifactMeta.finalMimeType()); + processedFilesExtension = fileExt; + fileCount++; + + String outputPath = writeLobToSiardStorage(zis, fileCount, fileExt); + siardPhysicalPaths.add(outputPath); + } + } + + double lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeTotal); + + if (fileCount > 0) { + writeLobReferenceToXml(columnIndex); + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + report.originalInputFile(), processedFilesExtension, null); + } else { + whiteNilCell(columnIndex); + } + + // Enriquecimento e Auditoria + ConversionReport enrichedReport = report + .withContext(new DbptkContext(tableCounter, columnIndex, siardPhysicalPaths)); + auditor.appendAuditRecord(enrichedReport); + + } catch (Exception e) { + throw new ModuleException().withMessage("Failed to process converted ZIP archive").withCause(e); + } + } + + private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) { + return mapper.readValue(zis.readAllBytes(), ConversionReport.class); + } + } + } + return null; + } + + private ArtifactReport findArtifactMetadata(List artifacts, String fileName) { + if (artifacts == null) + return null; + return artifacts.stream().filter(a -> fileName.equals(a.logicalName())).findFirst().orElse(null); + } + + private String writeLobToSiardStorage(InputStream zis, int fileCount, String extension) throws Exception { + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + fileCount + "." + extension; + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, writeStrategy); + zis.transferTo(out); + SIARDDKFileIndexFileStrategy.addFile(outputPath); + return outputPath; + } + + private void writeLobReferenceToXml(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + } + private void whiteNilCell(int columnIndex) throws IOException { tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java new file mode 100644 index 000000000..a5d4d30d1 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java @@ -0,0 +1,41 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Enriches and persists LOB conversion reports. + * + * @author Gabriel Barros + */ +public class LobConversionAuditor { + private static final Logger logger = LoggerFactory.getLogger(LobConversionAuditor.class); + private final ObjectMapper mapper; + private final Path auditFilePath; + + public LobConversionAuditor(Path baseExportDirectory, String archiveName) { + this.mapper = new ObjectMapper(); + String fileName = archiveName + "_lob_conversion_audit.jsonl"; + this.auditFilePath = baseExportDirectory.resolve(fileName); + } + + public void appendAuditRecord(ConversionReport report) { + if (report == null) + return; + + try { + String jsonLine = mapper.writeValueAsString(report); + Files.writeString(auditFilePath, jsonLine + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + logger.error("Failed to append conversion report to audit log.", e); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java new file mode 100644 index 000000000..c774cfd33 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java @@ -0,0 +1,13 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ArtifactReport(String logicalName, String originalMimeType, String finalMimeType, boolean isBypassed, + List formatHistory, List auditTrail, String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java new file mode 100644 index 000000000..0c1307ab7 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java @@ -0,0 +1,13 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record AuditTrailStep(String stepId, String pluginId, String agentName, String agentVersion, String agentType, + String command, Map parameters, long durationMs, boolean successful, String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java new file mode 100644 index 000000000..363d24e53 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -0,0 +1,17 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ConversionReport(String jobId, String status, String originalInputFile, Integer totalArtifactsProduced, + List artifacts, String errorMessage, DbptkContext dbptkContext) { + public ConversionReport withContext(DbptkContext context) { + return new ConversionReport(jobId, status, originalInputFile, totalArtifactsProduced, artifacts, errorMessage, + context); + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java new file mode 100644 index 000000000..bf83dee64 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java @@ -0,0 +1,9 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +/** + * @author Gabriel Barros + */ +public record DbptkContext(int tableIndex, int columnIndex, List siardPaths) { +} From dd92ad18e6185783658e77e59ef6203894c30cad Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Mon, 22 Jun 2026 11:04:21 +0100 Subject: [PATCH 21/37] Add ComplianceStatus enum and enhance report models with additional properties --- .../content/SIARDDKContentExportStrategy.java | 10 +-- .../output/SIARDDKDatabaseExportModule.java | 5 +- .../conversion/HttpLobConversionService.java | 62 +------------------ .../model/report/ArtifactReport.java | 8 ++- .../model/report/AuditTrailStep.java | 8 ++- .../model/report/ComplianceStatus.java | 8 +++ .../model/report/ConversionReport.java | 11 +++- .../conversion/model/report/DbptkContext.java | 5 +- 8 files changed, 43 insertions(+), 74 deletions(-) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index a796e4086..d5f227166 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -337,7 +337,7 @@ public Row tableRow(Row row) throws ModuleException { // BLOB EXTRACTION DELEGATION // ------------------------------------------------------------- if (mimeType.equals("application/zip")) { - processConvertedLobArchive(binaryCell, columnIndex); + processConvertedLobArchive(binaryCell, row.getIndex(), columnIndex); } else { logger.warn( "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", @@ -359,7 +359,8 @@ public Row tableRow(Row row) throws ModuleException { return row; } - private void processConvertedLobArchive(BinaryCell binaryCell, int columnIndex) throws ModuleException { + private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, int columnIndex) + throws ModuleException { try { ConversionReport report = extractReportFromZip(binaryCell); if (report == null) { @@ -399,14 +400,13 @@ private void processConvertedLobArchive(BinaryCell binaryCell, int columnIndex) if (fileCount > 0) { writeLobReferenceToXml(columnIndex); SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - report.originalInputFile(), processedFilesExtension, null); + report.originalFilename(), processedFilesExtension, null); } else { whiteNilCell(columnIndex); } - // Enriquecimento e Auditoria ConversionReport enrichedReport = report - .withContext(new DbptkContext(tableCounter, columnIndex, siardPhysicalPaths)); + .withContext(new DbptkContext(tableCounter, rowIndex, columnIndex, siardPhysicalPaths)); auditor.appendAuditRecord(enrichedReport); } catch (Exception e) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 06b2e6b5b..9fd917c1e 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -367,7 +367,10 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { // Track extracted parts to clean them individually later transientPaths.addAll(result.convertedFiles()); transientPaths.add(result.reportFile()); - transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + transientPaths.add(result.zipFile()); + if (!result.convertedFiles().isEmpty()) { + transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + } } catch (IOException | InterruptedException | HttpLobConversionServiceException e) { String statusCodeInfo = ""; if (e instanceof HttpLobConversionServiceException apiEx && apiEx.getHttpStatusCode() != null) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 83256da38..bf20cadd6 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -146,34 +146,13 @@ private void waitForCompletion(String cellId, String jobId) throw new HttpLobConversionServiceException("Timeout after waiting for conversion of cell: " + cellId); } - /** - * Downloads the resulting ZIP and extracts its contents, freeing the ZIP file - * immediately after. - */ - private ConversionResult downloadAndExtractResult(String cellId, String jobId) throws Exception { - Path tempZipFile = Files.createTempFile("siarddk_conv_" + cellId + "_", ".zip"); - fileTracker.track(tempZipFile); - - try { - HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) - .GET().build(); - - executeWithRetry(downloadRequest, BodyHandlers.ofFile(tempZipFile), MAX_NETWORK_RETRIES); - - return extractZipContents(cellId, tempZipFile); - } finally { - // Free disk space immediately after extraction - fileTracker.deleteEarly(tempZipFile); - } - } - /** * Downloads the resulting ZIP and lists its contents, returning the compressed * file. */ private ConversionResult downloadResult(String cellId, String jobId) throws IOException, InterruptedException { - Path zipFile = Files.createFile(Path.of("siarddk_conv_" + cellId + "_" + jobId + ".zip")); - // TODO: Should this still be tracked? + Path zipFile = Files.createTempFile("siarddk_conv_" + cellId + "_" + jobId, ".zip"); + fileTracker.track(zipFile); HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) @@ -184,43 +163,6 @@ private ConversionResult downloadResult(String cellId, String jobId) throws IOEx return listZipContents(cellId, zipFile); } - private ConversionResult extractZipContents(String cellId, Path tempZipFile) throws Exception { - Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); - fileTracker.trackDir(extractionDir); - - Path normalizedExtractionDir = extractionDir.normalize(); - - List convertedFiles = new ArrayList<>(); - Path reportFile = null; - - try (ZipInputStream zis = new ZipInputStream(new FileInputStream(tempZipFile.toFile()))) { - ZipEntry zipEntry; - while ((zipEntry = zis.getNextEntry()) != null) { - Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); - - if (!extractedFilePath.startsWith(normalizedExtractionDir)) { - throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); - } - - if (!zipEntry.isDirectory()) { - Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); - - if (zipEntry.getName().toLowerCase().contains("report")) { - reportFile = extractedFilePath; - } else { - convertedFiles.add(extractedFilePath); - } - } - } - } - - if (convertedFiles.isEmpty() || reportFile == null) { - throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); - } - - return new ConversionResult(convertedFiles, reportFile, tempZipFile); - } - private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java index c774cfd33..80552d0b5 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java @@ -3,11 +3,15 @@ import java.util.List; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; /** * @author Gabriel Barros */ @JsonIgnoreProperties(ignoreUnknown = true) -public record ArtifactReport(String logicalName, String originalMimeType, String finalMimeType, boolean isBypassed, - List formatHistory, List auditTrail, String errorMessage) { +public record ArtifactReport(@JsonProperty("logicalName") String logicalName, + @JsonProperty("originalMimeType") String originalMimeType, @JsonProperty("finalMimeType") String finalMimeType, + @JsonProperty("isBypassed") boolean isBypassed, @JsonProperty("complianceStatus") ComplianceStatus complianceStatus, + @JsonProperty("formatHistory") List formatHistory, + @JsonProperty("auditTrail") List auditTrail, @JsonProperty("errorMessage") String errorMessage) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java index 0c1307ab7..eab47c711 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java @@ -3,11 +3,15 @@ import java.util.Map; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; /** * @author Gabriel Barros */ @JsonIgnoreProperties(ignoreUnknown = true) -public record AuditTrailStep(String stepId, String pluginId, String agentName, String agentVersion, String agentType, - String command, Map parameters, long durationMs, boolean successful, String errorMessage) { +public record AuditTrailStep(@JsonProperty("stepId") String stepId, @JsonProperty("pluginId") String pluginId, + @JsonProperty("agentName") String agentName, @JsonProperty("agentVersion") String agentVersion, + @JsonProperty("agentType") String agentType, @JsonProperty("command") String command, + @JsonProperty("parameters") Map parameters, @JsonProperty("durationMs") long durationMs, + @JsonProperty("successful") boolean successful, @JsonProperty("errorMessage") String errorMessage) { } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java new file mode 100644 index 000000000..8d484c56a --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +/** + * @author Gabriel Barros + */ +public enum ComplianceStatus { + PASSED, PARTIAL, FAILED +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java index 363d24e53..4deafef2c 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -2,16 +2,21 @@ import java.util.List; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; /** * @author Gabriel Barros */ @JsonIgnoreProperties(ignoreUnknown = true) -public record ConversionReport(String jobId, String status, String originalInputFile, Integer totalArtifactsProduced, - List artifacts, String errorMessage, DbptkContext dbptkContext) { +public record ConversionReport(@JsonProperty("jobId") String jobId, @JsonProperty("status") JobStatus status, + @JsonProperty("originalFilename") String originalFilename, + @JsonProperty("totalArtifactsProduced") Integer totalArtifactsProduced, + @JsonProperty("artifacts") List artifacts, @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("dbptkContext") DbptkContext dbptkContext) { public ConversionReport withContext(DbptkContext context) { - return new ConversionReport(jobId, status, originalInputFile, totalArtifactsProduced, artifacts, errorMessage, + return new ConversionReport(jobId, status, originalFilename, totalArtifactsProduced, artifacts, errorMessage, context); } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java index bf83dee64..900387b56 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java @@ -2,8 +2,11 @@ import java.util.List; +import com.fasterxml.jackson.annotation.JsonProperty; + /** * @author Gabriel Barros */ -public record DbptkContext(int tableIndex, int columnIndex, List siardPaths) { +public record DbptkContext(@JsonProperty("tableIndex") int tableIndex, @JsonProperty("rowIndex") long rowIndex, + @JsonProperty("columnIndex") int columnIndex, @JsonProperty("siardPaths") List siardPaths) { } From 643e5b933750d8f684268b0e759f0ae1776ebc76 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Mon, 22 Jun 2026 15:58:02 +0100 Subject: [PATCH 22/37] Include researchIndex.xsd in produced SIARDDK 128 --- .../siard/common/path/SIARDDKMetadataPathStrategy.java | 7 ++++--- .../modules/siard/constants/SIARDDKConstants.java | 1 + .../siard/out/metadata/SIARDDKMetadataExportStrategy.java | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java index a4fe44a1a..f0a445522 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java @@ -7,12 +7,12 @@ */ package com.databasepreservation.modules.siard.common.path; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; - import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.Arrays; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + /** * @author Andreas Kring * @@ -54,7 +54,8 @@ public boolean checkFilename(String filename) { // Valid filenames String[] validFileNames = {SIARDDKConstants.TABLE_INDEX, SIARDDKConstants.ARCHIVE_INDEX, SIARDDKConstants.DOC_INDEX, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, SIARDDKConstants.FILE_INDEX, - SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, "fileIndex_original", "docIndex_original"}; + SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, SIARDDKConstants.RESEARCH_INDEX, + "fileIndex_original", "docIndex_original"}; ArrayList validFilenames = new ArrayList(Arrays.asList(validFileNames)); if (validFilenames.contains(filename)) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java index 68dd9d90b..203433413 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java @@ -54,6 +54,7 @@ public class SIARDDKConstants { public static final String ARCHIVE_INDEX = "archiveIndex"; public static final String TABLE_INDEX = "tableIndex"; public static final String FILE_INDEX = "fileIndex"; + public static final String RESEARCH_INDEX = "researchIndex"; public static final String DOC_INDEX = "docIndex"; public static final String DOCUMENT_IDENTIFICATION = "documentIdentification"; public static final String XML_SCHEMA = "XMLSchema"; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index 7fee0e106..f728ac0d8 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -154,6 +154,7 @@ public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContaine writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); if (lobsTracker.getLOBsCount() > 0) { writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); } From 88435918b7a1ac9a625c85d0201d3d865c5df183 Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Tue, 23 Jun 2026 09:18:39 +0100 Subject: [PATCH 23/37] Only include researchIndex.xsd in SIARDDK128 --- .../SIARDDK128MetadataExportStrategy.java | 16 ++++++++++++ .../SIARDDKMetadataExportStrategy.java | 26 +++++++++---------- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java index 4a0b82c5f..5b28bd2d3 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -106,4 +106,20 @@ public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContaine createLocalSharedFolder(outputContainer); } + + @Override + public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + + // Write contents to Schemas/standard + writeSchemaFile(outputContainer, SIARDDKConstants.XML_SCHEMA, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.TABLE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); + if (lobsTracker.getLOBsCount() > 0) { + writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); + } + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index f728ac0d8..53de790e0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -7,6 +7,18 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.reporters.Reporter; import com.databasepreservation.model.structure.DatabaseStructure; @@ -17,17 +29,6 @@ import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.write.WriteStrategy; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; /** * @author Andreas Kring @@ -154,7 +155,6 @@ public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContaine writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); - writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); if (lobsTracker.getLOBsCount() > 0) { writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); } @@ -165,7 +165,7 @@ public void setOnceReporter(Reporter reporter) { this.reporter = reporter; } - private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) + protected void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) throws ModuleException { InputStream inputStream = this.getClass().getResourceAsStream(metadataPathStrategy.getXsdResourcePath(indexFile)); From 314a0236f96083a9ffd2b05a693356f7fb4fd08d Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Mon, 22 Jun 2026 15:59:52 +0100 Subject: [PATCH 24/37] Enhance SQLServerDatatypeImporter to support BLOB type handling for varbinary columns --- .../sqlserver/in/SQLServerDatatypeImporter.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java index 400746220..0efb41e01 100644 --- a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java +++ b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java @@ -34,6 +34,19 @@ protected Type getBinaryType(String typeName, int columnSize, int decimalDigits, if (typeName.contains("timestamp") || typeName.contains("rowversion")) { return super.getBinaryType(typeName, 8, decimalDigits, numPrecRadix); } + return super.getBinaryType(typeName, columnSize, decimalDigits, numPrecRadix); } + + @Override + protected Type getVarbinaryType(String typeName, int columnSize, int decimalDigits, int numPrecRadix) { + if (typeName.equalsIgnoreCase("varbinary") && (columnSize == 2147483647 || columnSize == -1)) { + Type type = new SimpleTypeBinary(columnSize); + type.setSql99TypeName("BINARY LARGE OBJECT"); + type.setSql2008TypeName("BINARY LARGE OBJECT"); + return type; + } + + return super.getVarbinaryType(typeName, columnSize, decimalDigits, numPrecRadix); + } } From 789ff8dfc2e86b4898722fb9591feb680a4129b1 Mon Sep 17 00:00:00 2001 From: Gabriel Barros Date: Thu, 25 Jun 2026 10:07:46 +0100 Subject: [PATCH 25/37] Add ExportModuleContextManager for managing export module context and enhance DatabaseMigration and ExternalLOBSFilter for improved type handling --- .../DatabaseMigration.java | 107 ++++++++++-------- .../managers/ExportModuleContextManager.java | 43 +++++++ .../externalLobs/ExternalLOBSFilter.java | 24 +++- 3 files changed, 119 insertions(+), 55 deletions(-) create mode 100644 dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java diff --git a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java index 89548ca6f..11ac617e7 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java +++ b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java @@ -15,6 +15,7 @@ import org.apache.commons.lang3.tuple.Pair; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.modules.DatabaseImportModule; import com.databasepreservation.model.modules.DatabaseModuleFactory; @@ -60,67 +61,75 @@ public DatabaseImportModule getImportModule() throws ModuleException { public void migrate() throws ModuleException { validate(); - // get import module and export module instance - Map importParameters = buildParametersFromStringParameters(importModuleFactory, - importModuleFactoryStringParameters); - Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, - exportModuleFactoryStringParameters); - - DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); - DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); - - List beforeFilterModules = new ArrayList<>(); - List afterFilterModules = new ArrayList<>(); - for (int i = 0; i < filterFactories.size(); i++) { - Map filterParameters = new HashMap<>(); - if (!filterFactoriesStringParameters.isEmpty()) { - filterParameters = buildParametersFromStringParameters(filterFactories.get(i), - filterFactoriesStringParameters.get(i)); + try { + if (exportModuleFactory != null) { + ExportModuleContextManager.getInstance().setup(exportModuleFactory); } - if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { - afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); - } else { - beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + // get import module and export module instance + Map importParameters = buildParametersFromStringParameters(importModuleFactory, + importModuleFactoryStringParameters); + Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, + exportModuleFactoryStringParameters); + + DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); + DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); + + List beforeFilterModules = new ArrayList<>(); + List afterFilterModules = new ArrayList<>(); + for (int i = 0; i < filterFactories.size(); i++) { + Map filterParameters = new HashMap<>(); + if (!filterFactoriesStringParameters.isEmpty()) { + filterParameters = buildParametersFromStringParameters(filterFactories.get(i), + filterFactoriesStringParameters.get(i)); + } + + if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { + afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } else { + beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } } - } - // set reporters - importModule.setOnceReporter(reporter); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - filterModule.setOnceReporter(reporter); - } + // set reporters + importModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : beforeFilterModules) { + filterModule.setOnceReporter(reporter); + } - for (DatabaseFilterModule filterModule : afterFilterModules) { - filterModule.setOnceReporter(reporter); - } - for (DatabaseFilterModule filterModule : filterModules) { - filterModule.setOnceReporter(reporter); - } - exportModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : afterFilterModules) { + filterModule.setOnceReporter(reporter); + } + for (DatabaseFilterModule filterModule : filterModules) { + filterModule.setOnceReporter(reporter); + } + exportModule.setOnceReporter(reporter); - // create module chain with filters in the middle - Collections.reverse(filterModules); - Collections.reverse(beforeFilterModules); - Collections.reverse(afterFilterModules); + // create module chain with filters in the middle + Collections.reverse(filterModules); + Collections.reverse(beforeFilterModules); + Collections.reverse(afterFilterModules); - DatabaseFilterModule sinkModule = new SinkModule(); + DatabaseFilterModule sinkModule = new SinkModule(); - for (DatabaseFilterModule filterModule : afterFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : afterFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - sinkModule = exportModule.migrateDatabaseTo(sinkModule); + sinkModule = exportModule.migrateDatabaseTo(sinkModule); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : beforeFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - for (DatabaseFilterModule filterModule : filterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : filterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - importModule.migrateDatabaseTo(sinkModule); + importModule.migrateDatabaseTo(sinkModule); + } finally { + ExportModuleContextManager.destroy(); + } } /** diff --git a/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java new file mode 100644 index 000000000..ffe7da3ef --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java @@ -0,0 +1,43 @@ +package com.databasepreservation.managers; + +import com.databasepreservation.model.exception.UnsupportedModuleException; +import com.databasepreservation.model.modules.DatabaseModuleFactory; +import com.databasepreservation.model.parameters.Parameters; + +/** + * @author Gabriel Barros + */ +public class ExportModuleContextManager { + private static ExportModuleContextManager instance = null; + private String moduleName; + private Parameters exportModuleParameters; + + public static ExportModuleContextManager getInstance() { + if (instance == null) { + instance = new ExportModuleContextManager(); + } + + return instance; + } + + public static void destroy() { + instance = null; + } + + public void setup(DatabaseModuleFactory exportModuleFactory) throws UnsupportedModuleException { + moduleName = exportModuleFactory.getModuleName(); + exportModuleParameters = exportModuleFactory.getExportModuleParameters(); + } + + public String getModuleName() { + return moduleName; + } + + public Parameters getExportModuleParameters() { + return exportModuleParameters; + } + + public boolean isSiadDKModule() { + return moduleName.contains("siard-dk"); + } +} diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java index bd34df3ab..50a99a33b 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java @@ -14,9 +14,11 @@ import java.util.Map; import java.util.Set; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.managers.ModuleConfigurationManager; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.NullCell; @@ -93,12 +95,7 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException Type original = column.getType(); description.append(". Original description: '").append(original.getDescription()).append("')"); - SimpleTypeBinary newType = new SimpleTypeBinary(); - newType.setSql99TypeName("BINARY VARYING", 1); - newType.setSql2008TypeName("BINARY VARYING", 1); - newType.setOriginalTypeName(original.getOriginalTypeName()); - newType.setOutsideDatabase(true); - + SimpleTypeBinary newType = getSimpleTypeBinary(original); column.setType(newType); column.setDescription(description.toString()); } @@ -110,6 +107,21 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException this.exportModule.handleStructure(structure); } + @NotNull + private static SimpleTypeBinary getSimpleTypeBinary(Type original) { + SimpleTypeBinary newType = new SimpleTypeBinary(); + if (ExportModuleContextManager.getInstance().isSiadDKModule()) { + newType.setSql99TypeName("BINARY LARGE OBJECT"); + newType.setSql2008TypeName("BINARY LARGE OBJECT"); + } else { + newType.setSql99TypeName("BINARY VARYING", 1); + newType.setSql2008TypeName("BINARY VARYING", 1); + } + newType.setOriginalTypeName(original.getOriginalTypeName()); + newType.setOutsideDatabase(true); + return newType; + } + @Override public void handleDataOpenSchema(String schemaName) throws ModuleException { this.exportModule.handleDataOpenSchema(schemaName); From 4dc9360cc31d8e75f87943e0eff2eadf3edd9741 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Wed, 8 Jul 2026 14:11:29 +0100 Subject: [PATCH 26/37] refactor: cell value strip trailing --- .../ExternalLOBSCellHandlerFileSystem.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java index 162692682..2c59e4474 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java @@ -40,7 +40,9 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { return newCell; } - Path blobPath = basePath.resolve(cellValue); + String cellValueStripTrailing = cellValue.stripTrailing(); + + Path blobPath = basePath.resolve(cellValueStripTrailing); if (Files.exists(blobPath)) { if (Files.isRegularFile(blobPath)) { @@ -48,15 +50,15 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { newCell = new BinaryCell(cellId, new PathInputStreamProvider(blobPath)); } catch (ModuleException e) { reporter.ignored("Cell " + cellId, - blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, - blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, "Path: " + blobPath.toString() + " could not be found; Base path: " - + this.basePath + " Cell Value: " + cellValue); + + this.basePath + " Cell Value: " + cellValueStripTrailing); } return newCell; } From 0dca27b06abde1c844569a4cfc2e6b9919887244 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Thu, 30 Jul 2026 12:09:40 +0100 Subject: [PATCH 27/37] fix: original filename of external lobs on doc indexes --- .../com/databasepreservation/model/data/BinaryCell.java | 4 ++++ .../modules/externalLobs/ExternalLOBSFilter.java | 4 ++++ .../siard/out/content/SIARDDKContentExportStrategy.java | 7 +++++++ .../siard/out/output/SIARDDKDatabaseExportModule.java | 3 ++- .../services/conversion/model/report/ConversionReport.java | 5 +++++ 5 files changed, 22 insertions(+), 1 deletion(-) diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java index 63f8fbf3b..a5a028314 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java @@ -139,4 +139,8 @@ public long getLength() { public String getMimeType() { return mimeType; } + + public void setFile(String file) { + this.file = file; + } } diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java index 50a99a33b..b64d47e6a 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java @@ -20,6 +20,7 @@ import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.managers.ModuleConfigurationManager; +import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.NullCell; import com.databasepreservation.model.data.Row; @@ -167,6 +168,9 @@ public void handleDataRow(Row row) throws ModuleException { .get(currentTable.getId() + index); Cell newCell = getExternalLOBSCellHandler(externalLobsConfiguration).handleCell(cell.getId(), simpleCell.getSimpleData()); + if (newCell instanceof BinaryCell binaryCell) { + binaryCell.setFile(simpleCell.getSimpleData()); + } rowCells.set(index, newCell); } else { reporter.ignored("Cell " + cell.getId(), "reference to external LOB is null"); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index d5f227166..0a497a3ce 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -20,6 +20,7 @@ import java.util.zip.ZipInputStream; import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.jdom2.Document; import org.jdom2.Element; @@ -367,6 +368,12 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in throw new ModuleException().withMessage("Missing conversion_report.json in cell archive."); } + String fileFromCell = binaryCell.getFile(); + if (fileFromCell != null) { + String filename = FilenameUtils.getName(fileFromCell).stripTrailing(); + report = report.withOriginalFilename(filename); + } + List siardPhysicalPaths = new ArrayList<>(); int fileCount = 0; String processedFilesExtension = "tif"; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index 9fd917c1e..d750ca8c2 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -360,8 +360,9 @@ private ProcessedRowContext processRowAsync(Row row) throws ModuleException { // TODO: Handle multiple files per cell if needed. Currently assumes single file // output. - Cell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), + BinaryCell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), "application/zip"); + newCell.setFile(binCell.getFile()); cells.set(i, newCell); // Track extracted parts to clean them individually later diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java index 4deafef2c..32aaec45f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -19,4 +19,9 @@ public ConversionReport withContext(DbptkContext context) { return new ConversionReport(jobId, status, originalFilename, totalArtifactsProduced, artifacts, errorMessage, context); } + + public ConversionReport withOriginalFilename(String newFilename) { + return new ConversionReport(jobId, status, newFilename, totalArtifactsProduced, artifacts, errorMessage, + dbptkContext); + } } From 115d95b0527d40b184e30371a9bb4a44e209b9ba Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Wed, 9 Sep 2026 13:54:21 +0100 Subject: [PATCH 28/37] fix: issues after rebase --- .../content/SIARDDKContentExportStrategy.java | 39 ++++++++++++++++++- .../SIARDDK1007DatabaseExportModule.java | 5 ++- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 0a497a3ce..136e945e5 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -40,6 +40,7 @@ import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.SchemaStructure; import com.databasepreservation.model.structure.TableStructure; +import com.databasepreservation.modules.siard.common.LargeObject; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDConstants; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; @@ -343,7 +344,9 @@ public Row tableRow(Row row) throws ModuleException { logger.warn( "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", mimeType, tableCounter, columnIndex); - whiteNilCell(columnIndex); + //whiteNilCell(columnIndex); + + archiveRawLob(binaryCell, columnIndex); } } else { // never happens @@ -421,6 +424,40 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in } } + private void archiveRawLob(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + String fileExtension; + if (mimetypeHandler.isMimetypeAllowed(mimeType)) { + fileExtension = mimetypeHandler.getFileExtension(mimeType); + } else { + fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; + foundUnknownMimetype = true; + } + + double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); + + String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; + LargeObject blob = new LargeObject(binaryCell, path); + + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + InputStream in = blob.getInputStreamProvider().createInputStream(); + IOUtils.copy(in, out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(out); + blob.getInputStreamProvider().cleanResources(); + + lobsTracker.addLOB(lobSizeMB); + + writeLobReferenceToXml(columnIndex); + + String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() + : "originalFilename"; + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFilename, fileExtension, null); + + SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + } + private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java index e775df706..cac3d926d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java @@ -7,9 +7,10 @@ */ package com.databasepreservation.modules.siard.out.output; -import com.databasepreservation.modules.siard.bindings.siard_dk_1007.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import dk.sa.xmlns.diark._1_0.fileindex.FileIndexType; + /** * @author António Lindo * @@ -27,6 +28,6 @@ String getJAXBContext() { @Override Class getJAXBContextClass() { - return SiardDiark.class; + return FileIndexType.class; } } From 5852f4bc969228a650896754bc6ed622aa54a615 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Tue, 15 Sep 2026 13:17:24 +0100 Subject: [PATCH 29/37] refactor: non normalizer blobs and siard tests --- .../testing/integration/siard/SiardTest.java | 7 +- .../content/SIARDDKContentExportStrategy.java | 78 +++++++++---------- 2 files changed, 42 insertions(+), 43 deletions(-) diff --git a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java index 42a7f4db0..33f62813b 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java +++ b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java @@ -36,6 +36,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.databasepreservation.common.io.providers.TemporaryPathInputStreamProvider; import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.Row; @@ -539,15 +540,15 @@ protected DatabaseStructure generateDatabaseStructure() throws ModuleException, new Row(1, Arrays.asList(new SimpleCell("table02.col121.0", "1"), new SimpleCell("table02.col122.0", "3"), new SimpleCell("table02.col123.0", "abc"), new SimpleCell("table02.col124.0", "def"), - new BinaryCell("table02.col125.0", newBlob()))), + new BinaryCell("table02.col125.0", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(2, Arrays.asList(new SimpleCell("table02.col121.1", "2"), new SimpleCell("table02.col122.1", "1"), new SimpleCell("table02.col123.1", "dns"), new SimpleCell("table02.col124.1", "dud"), - new BinaryCell("table02.col125.1", newBlob()))), + new BinaryCell("table02.col125.1", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(3, Arrays.asList(new SimpleCell("table02.col121.2", "3"), new SimpleCell("table02.col122.2", "2"), new SimpleCell("table02.col123.2", "usl"), new SimpleCell("table02.col124.2", "aps"), - new BinaryCell("table02.col125.2", newBlob()))))); + new BinaryCell("table02.col125.2", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))))); tableRows.put("schema02.table01", new ArrayList()); tableRows.put("schema02.table02", new ArrayList()); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 136e945e5..fdb6e9866 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -341,12 +341,7 @@ public Row tableRow(Row row) throws ModuleException { if (mimeType.equals("application/zip")) { processConvertedLobArchive(binaryCell, row.getIndex(), columnIndex); } else { - logger.warn( - "Found BLOB with unsupported mimetype '{}' in table {}, column {}. ignoring content and archiving as .bin file.", - mimeType, tableCounter, columnIndex); - //whiteNilCell(columnIndex); - - archiveRawLob(binaryCell, columnIndex); + processRawLobFile(binaryCell, columnIndex); } } else { // never happens @@ -363,6 +358,43 @@ public Row tableRow(Row row) throws ModuleException { return row; } + private void processRawLobFile(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + String fileExtension; + if (mimetypeHandler.isMimetypeAllowed(mimeType)) { + fileExtension = mimetypeHandler.getFileExtension(mimeType); + } else { + logger.warn( + "Found BLOB with unsupported mimetype '{}' in table {}, column {}. archiving as .bin file.", + mimeType, tableCounter, columnIndex); + fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; + foundUnknownMimetype = true; + } + + double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); + + String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; + LargeObject blob = new LargeObject(binaryCell, path); + + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + InputStream in = blob.getInputStreamProvider().createInputStream(); + IOUtils.copy(in, out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(out); + blob.getInputStreamProvider().cleanResources(); + + lobsTracker.addLOB(lobSizeMB); + + writeLobReferenceToXml(columnIndex); + + String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() + : "originalFilename"; + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFilename, fileExtension, null); + + SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + } + private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, int columnIndex) throws ModuleException { try { @@ -424,40 +456,6 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in } } - private void archiveRawLob(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { - String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; - String fileExtension; - if (mimetypeHandler.isMimetypeAllowed(mimeType)) { - fileExtension = mimetypeHandler.getFileExtension(mimeType); - } else { - fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; - foundUnknownMimetype = true; - } - - double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - - String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; - LargeObject blob = new LargeObject(binaryCell, path); - - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); - InputStream in = blob.getInputStreamProvider().createInputStream(); - IOUtils.copy(in, out); - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(out); - blob.getInputStreamProvider().cleanResources(); - - lobsTracker.addLOB(lobSizeMB); - - writeLobReferenceToXml(columnIndex); - - String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() - : "originalFilename"; - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - originalFilename, fileExtension, null); - - SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); - } - private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { ZipEntry zipEntry; From d74525888329f1daf203d20da9c208408ee2336d Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Tue, 15 Sep 2026 17:40:13 +0100 Subject: [PATCH 30/37] feat: siard dk mime types coverage --- .../out/content/SIARDDKMimetypeHandler.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java index 7f960b37c..c671fb30c 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java @@ -25,11 +25,28 @@ public class SIARDDKMimetypeHandler implements MimetypeHandler { public SIARDDKMimetypeHandler() { mimetypeMap = new HashMap(); - mimetypeMap.put("image/tiff", "tif"); mimetypeMap.put("image/jp2", "jp2"); + mimetypeMap.put("image/jpeg2000", "jp2"); + mimetypeMap.put("image/x-jp2", "jp2"); mimetypeMap.put("audio/mpeg", "mp3"); + mimetypeMap.put("audio/mp3", "mp3"); + mimetypeMap.put("audio/x-mpeg", "mp3"); + mimetypeMap.put("audio/mpeg3", "mp3"); mimetypeMap.put("video/mp4", "mpg"); mimetypeMap.put("video/mp2t", "mpg"); + mimetypeMap.put("video/mpeg2", "mpg"); + mimetypeMap.put("video/x-mpeg2", "mpg"); + mimetypeMap.put("video/mpeg", "mpg"); + mimetypeMap.put("video/mpeg4-generic", "mpg"); + mimetypeMap.put("video/x-m4v", "mpg"); + mimetypeMap.put("application/mp4", "mpg"); + mimetypeMap.put("application/gml+xml", "gml"); + mimetypeMap.put("text/gml", "gml"); + mimetypeMap.put("application/xml", "gml"); + mimetypeMap.put("audio/wav", "wav"); + mimetypeMap.put("audio/x-wav", "wav"); + mimetypeMap.put("audio/wave", "wav"); + mimetypeMap.put("audio/vnd.wave", "wav"); // TO-DO: check mimetypes for MPEG with sa.dk // Wave files are missing in fileIndex.xsd - this is an error. Will be From 6d220551b1f332d72f3220ff53af6bc6a80b7313 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Tue, 15 Sep 2026 17:44:08 +0100 Subject: [PATCH 31/37] fix: tif format return --- .../modules/siard/out/content/SIARDDKMimetypeHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java index c671fb30c..57fc0c487 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java @@ -25,6 +25,7 @@ public class SIARDDKMimetypeHandler implements MimetypeHandler { public SIARDDKMimetypeHandler() { mimetypeMap = new HashMap(); + mimetypeMap.put("image/tiff", "tif"); mimetypeMap.put("image/jp2", "jp2"); mimetypeMap.put("image/jpeg2000", "jp2"); mimetypeMap.put("image/x-jp2", "jp2"); From a8ada7b5c756de2211effa8227c4609addf6c278 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Thu, 17 Sep 2026 11:39:31 +0100 Subject: [PATCH 32/37] refactor: removal of comments --- .../siard/out/content/SIARDDKMimetypeHandler.java | 9 --------- 1 file changed, 9 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java index 57fc0c487..09d732206 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java @@ -48,15 +48,6 @@ public SIARDDKMimetypeHandler() { mimetypeMap.put("audio/x-wav", "wav"); mimetypeMap.put("audio/wave", "wav"); mimetypeMap.put("audio/vnd.wave", "wav"); - // TO-DO: check mimetypes for MPEG with sa.dk - - // Wave files are missing in fileIndex.xsd - this is an error. Will be - // corrected by sa.dk later - - // mimetypeMap.put("audio/wav", "wav"); - // mimetypeMap.put("audio/x-wav", "wav"); - - // TO-DO: build (how?) GML files } /* From 227b153a707522930243ff62b39603dd1d2a6f79 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Wed, 16 Sep 2026 17:38:40 +0100 Subject: [PATCH 33/37] feat: bypass lobs reporter --- .../content/SIARDDKContentExportStrategy.java | 8 +++ .../conversion/BypassedLobReporter.java | 70 +++++++++++++++++++ .../model/report/BypassedLobEntry.java | 12 ++++ 3 files changed, 90 insertions(+) create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java create mode 100644 dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index fdb6e9866..5d08e708c 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -40,6 +40,7 @@ import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.SchemaStructure; import com.databasepreservation.model.structure.TableStructure; +import com.databasepreservation.modules.siard.SIARDDKModuleFactory; import com.databasepreservation.modules.siard.common.LargeObject; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDConstants; @@ -49,6 +50,7 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.databasepreservation.modules.siard.services.conversion.BypassedLobReporter; import com.databasepreservation.modules.siard.services.conversion.LobConversionAuditor; import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; @@ -79,6 +81,7 @@ public class SIARDDKContentExportStrategy implements ContentExportStrategy { private final MimetypeHandler mimetypeHandler; private final LobConversionAuditor auditor; + private final BypassedLobReporter bypassedLobReporter; private final ObjectMapper mapper; private Reporter reporter; @@ -102,6 +105,10 @@ public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { Path exportRoot = baseContainer.getPath().getParent(); String archiveName = baseContainer.getPath().getFileName().toString(); this.auditor = new LobConversionAuditor(exportRoot, archiveName); + + String targetLobFormat = siarddkExportModule.getExportModuleArgs() + .getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_TARGET_FORMAT, "image/tiff"); + this.bypassedLobReporter = new BypassedLobReporter(exportRoot, archiveName, targetLobFormat); } @Override @@ -450,6 +457,7 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in ConversionReport enrichedReport = report .withContext(new DbptkContext(tableCounter, rowIndex, columnIndex, siardPhysicalPaths)); auditor.appendAuditRecord(enrichedReport); + bypassedLobReporter.appendBypassedRecord(enrichedReport); } catch (Exception e) { throw new ModuleException().withMessage("Failed to process converted ZIP archive").withCause(e); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java new file mode 100644 index 000000000..976d22583 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java @@ -0,0 +1,70 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.BypassedLobEntry; +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * @author Vitor Leite + */ +public class BypassedLobReporter { + private static final Logger logger = LoggerFactory.getLogger(BypassedLobReporter.class); + private static final String REASON_CONVERSION_FAILED = "Conversion failed"; + private static final String REASON_FILTERED_FORMAT = "Retained the original format due to SIARD-DK specification"; + + private final ObjectMapper mapper; + private final Path reportFilePath; + private final String targetLobFormat; + + public BypassedLobReporter(Path baseExportDirectory, String archiveName, String targetLobFormat) { + this.mapper = new ObjectMapper(); + this.targetLobFormat = targetLobFormat; + String fileName = archiveName + "_bypassed_lob_report.jsonl"; + this.reportFilePath = baseExportDirectory.resolve(fileName); + } + + public void appendBypassedRecord(ConversionReport report) { + if (report == null || report.artifacts() == null) + return; + + for (ArtifactReport artifact : report.artifacts()) { + String reason = resolveBypassReason(artifact); + if (reason == null) + continue; + + BypassedLobEntry entry = new BypassedLobEntry(artifact.logicalName(), artifact.originalMimeType(), + artifact.finalMimeType(), artifact.isBypassed(), artifact.errorMessage(), reason, report.dbptkContext()); + append(entry); + } + } + + private String resolveBypassReason(ArtifactReport artifact) { + if (artifact.isBypassed()) { + return REASON_CONVERSION_FAILED; + } + + if (!targetLobFormat.equalsIgnoreCase(artifact.finalMimeType())) { + return REASON_FILTERED_FORMAT; + } + + return null; + } + + private void append(BypassedLobEntry entry) { + try { + String jsonLine = mapper.writeValueAsString(entry); + Files.writeString(reportFilePath, jsonLine + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + logger.error("Failed to append bypassed lob entry to report.", e); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java new file mode 100644 index 000000000..3ec0db906 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java @@ -0,0 +1,12 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Vitor Leite + */ +public record BypassedLobEntry(@JsonProperty("logicalName") String logicalName, + @JsonProperty("originalMimeType") String originalMimeType, @JsonProperty("finalMimeType") String finalMimeType, + @JsonProperty("isBypassed") boolean isBypassed, @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("reason") String reason, @JsonProperty("dbptkContext") DbptkContext dbptkContext) { +} From 137995e5d7453836c5db316df3724fc0a891966e Mon Sep 17 00:00:00 2001 From: Alexandre Flores Date: Fri, 4 Sep 2026 13:04:31 +0100 Subject: [PATCH 34/37] Fix off-by-one error in SiardDK lob paths --- .../out/content/SIARDDKContentExportStrategy.java | 13 ++++++++----- .../out/path/SIARDDKContentPathExportStrategy.java | 2 +- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 5d08e708c..a9707405e 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -379,6 +379,7 @@ private void processRawLobFile(BinaryCell binaryCell, int columnIndex) throws Mo } double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeMB); String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; LargeObject blob = new LargeObject(binaryCell, path); @@ -390,8 +391,6 @@ private void processRawLobFile(BinaryCell binaryCell, int columnIndex) throws Mo IOUtils.closeQuietly(out); blob.getInputStreamProvider().cleanResources(); - lobsTracker.addLOB(lobSizeMB); - writeLobReferenceToXml(columnIndex); String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() @@ -428,12 +427,19 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in ArtifactReport artifactMeta = findArtifactMetadata(report.artifacts(), zipEntry.getName()); + // Check if this file is bypassed if (artifactMeta == null || artifactMeta.isBypassed()) { logger.warn("Ignoring bypassed or unknown file: {}. Reason: {}", zipEntry.getName(), artifactMeta != null ? artifactMeta.errorMessage() : "Not in report"); continue; } + // LOB isn't bypassed; add it to tracker now + if (fileCount == 0) { + double lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeTotal); + } + String fileExt = mimetypeHandler.getFileExtension(artifactMeta.finalMimeType()); processedFilesExtension = fileExt; fileCount++; @@ -443,9 +449,6 @@ private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, in } } - double lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); - lobsTracker.addLOB(lobSizeTotal); - if (fileCount > 0) { writeLobReferenceToXml(columnIndex); SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java index 98731d8f1..6b197218d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java @@ -71,7 +71,7 @@ public String getBlobFilePath(int schemaIndex, int tableIndex, int columnIndex, // TO-DO: add test case int docCollectionCount = lobsTracker.getDocCollectionCount(); - int LOBsCount = lobsTracker.getLOBsCount() + 1; + int LOBsCount = lobsTracker.getLOBsCount(); // Note: code assumes one file in each folder return new StringBuilder().append(DOCUMENT_DIR).append(SIARDDKConstants.FILE_SEPARATOR).append(DOC_COLLECTION) From 739b1e9695dd48adaa8d9c59de4a9ad72547ec3d Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Thu, 17 Sep 2026 18:25:38 +0100 Subject: [PATCH 35/37] fix: db name processing --- .../out/metadata/TestSIARDDK1007TableIndexFileStrategy.java | 4 ++-- .../siard/out/metadata/SIARDDKTableIndexFileStrategy.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java b/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java index 644e84057..e90f70b9a 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java +++ b/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java @@ -9,10 +9,10 @@ import static org.testng.AssertJUnit.assertEquals; -import com.databasepreservation.modules.siard.common.adapters.SIARDDK1007Adapter; -import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter; import org.testng.annotations.Test; +import com.databasepreservation.modules.siard.common.adapters.SIARDDK1007Adapter; + /** * @author Andreas Kring * diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java index 07217ab04..fa1e18567 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java @@ -58,9 +58,9 @@ public Object generateXML(DatabaseStructure dbStructure) throws ModuleException // Set dbName - mandatory if (dbStructure.getDbOriginalName() != null) { - siarddkBinding.setDbName(dbStructure.getDbOriginalName()); + siarddkBinding.setDbName(escapeString(dbStructure.getDbOriginalName())); } else { - siarddkBinding.setDbName(dbStructure.getName()); + siarddkBinding.setDbName(escapeString(dbStructure.getName())); } // Set databaseProduct From 174c2e66535245f468b513725c0ae53db041f613 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Thu, 17 Sep 2026 17:58:38 +0100 Subject: [PATCH 36/37] feat: delete job from local storage after extraction --- .../conversion/HttpLobConversionService.java | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index bf20cadd6..49e749d2f 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -75,7 +75,9 @@ public ConversionResult convertLob(String cellId, InputStream inputStream) log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); waitForCompletion(cellId, jobId); - return downloadResult(cellId, jobId); + ConversionResult result = downloadResult(cellId, jobId); + deleteJob(cellId, jobId); + return result; } private String submitJob(String cellId, InputStream inputStream) @@ -163,6 +165,23 @@ private ConversionResult downloadResult(String cellId, String jobId) throws IOEx return listZipContents(cellId, zipFile); } + private void deleteJob(String cellId, String jobId) { + try { + HttpRequest deleteRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).DELETE().build(); + HttpResponse deleteResponse = executeWithRetry(deleteRequest, BodyHandlers.discarding(), + MAX_NETWORK_RETRIES); + + if (deleteResponse.statusCode() >= 400) { + log.warn("API rejected deletion of Job {} (Cell {}). Status: {}", jobId, cellId, deleteResponse.statusCode()); + } + } catch (IOException | InterruptedException e) { + if (e instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + log.warn("Failed to delete Job {} (Cell {}) after successful download: {}", jobId, cellId, e.getMessage()); + } + } + private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); fileTracker.trackDir(extractionDir); From c983d378664277362935a58fecc5bed689d6f102 Mon Sep 17 00:00:00 2001 From: VitorLelis Date: Fri, 18 Sep 2026 11:28:02 +0100 Subject: [PATCH 37/37] refactor: async delete and job status check --- .../conversion/HttpLobConversionService.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java index 49e749d2f..9dd6f3211 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -74,9 +74,11 @@ public ConversionResult convertLob(String cellId, InputStream inputStream) throws IOException, InterruptedException, HttpLobConversionServiceException { log.debug("Initiating conversion pipeline for cell: {}", cellId); String jobId = submitJob(cellId, inputStream); - waitForCompletion(cellId, jobId); + JobStatus status = waitForCompletion(cellId, jobId); ConversionResult result = downloadResult(cellId, jobId); - deleteJob(cellId, jobId); + if (JobStatus.COMPLETED.equals(status)) { + deleteJob(cellId, jobId); + } return result; } @@ -111,7 +113,7 @@ private String submitJob(String cellId, InputStream inputStream) return job.id(); } - private void waitForCompletion(String cellId, String jobId) + private JobStatus waitForCompletion(String cellId, String jobId) throws IOException, InterruptedException, HttpLobConversionServiceException { log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); @@ -125,7 +127,7 @@ private void waitForCompletion(String cellId, String jobId) switch (response.status()) { case JobStatus.COMPLETED -> { log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); - return; + return response.status(); } case JobStatus.FAILED, JobStatus.EVICTED -> { log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, @@ -166,20 +168,15 @@ private ConversionResult downloadResult(String cellId, String jobId) throws IOEx } private void deleteJob(String cellId, String jobId) { - try { - HttpRequest deleteRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).DELETE().build(); - HttpResponse deleteResponse = executeWithRetry(deleteRequest, BodyHandlers.discarding(), - MAX_NETWORK_RETRIES); + HttpRequest deleteRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).DELETE().build(); - if (deleteResponse.statusCode() >= 400) { - log.warn("API rejected deletion of Job {} (Cell {}). Status: {}", jobId, cellId, deleteResponse.statusCode()); + httpClient.sendAsync(deleteRequest, BodyHandlers.discarding()).whenComplete((response, e) -> { + if (response.statusCode() >= 400) { + log.warn("API rejected deletion of Job {} (Cell {}). Status: {}", jobId, cellId, response.statusCode()); + } else { + log.warn("Failed to delete Job {} (Cell {}) after successful download: {}", jobId, cellId, e.getMessage()); } - } catch (IOException | InterruptedException e) { - if (e instanceof InterruptedException) { - Thread.currentThread().interrupt(); - } - log.warn("Failed to delete Job {} (Cell {}) after successful download: {}", jobId, cellId, e.getMessage()); - } + }); } private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException {