diff --git a/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java b/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java index 644e84057..e90f70b9a 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java +++ b/dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java @@ -9,10 +9,10 @@ import static org.testng.AssertJUnit.assertEquals; -import com.databasepreservation.modules.siard.common.adapters.SIARDDK1007Adapter; -import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter; import org.testng.annotations.Test; +import com.databasepreservation.modules.siard.common.adapters.SIARDDK1007Adapter; + /** * @author Andreas Kring * diff --git a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java index eb2a56cc7..b2d8be941 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java +++ b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java @@ -11,6 +11,9 @@ import java.util.ArrayList; import java.util.Iterator; +import com.databasepreservation.Main; +import com.databasepreservation.utils.ConfigUtils; +import org.testng.annotations.BeforeClass; import org.testng.annotations.DataProvider; import org.testng.annotations.Test; @@ -30,6 +33,11 @@ @Test(groups = {"siarddk-roundtrip"}) public class SiardDKTestWrapper { + @BeforeClass + public void init() { + ConfigUtils.initialize(); + } + @DataProvider public Iterator siardVersionsProvider() { ArrayList tests = new ArrayList(); diff --git a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java index 42a7f4db0..33f62813b 100644 --- a/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java +++ b/dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java @@ -36,6 +36,7 @@ import org.testng.annotations.DataProvider; import org.testng.annotations.Test; +import com.databasepreservation.common.io.providers.TemporaryPathInputStreamProvider; import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.Row; @@ -539,15 +540,15 @@ protected DatabaseStructure generateDatabaseStructure() throws ModuleException, new Row(1, Arrays.asList(new SimpleCell("table02.col121.0", "1"), new SimpleCell("table02.col122.0", "3"), new SimpleCell("table02.col123.0", "abc"), new SimpleCell("table02.col124.0", "def"), - new BinaryCell("table02.col125.0", newBlob()))), + new BinaryCell("table02.col125.0", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(2, Arrays.asList(new SimpleCell("table02.col121.1", "2"), new SimpleCell("table02.col122.1", "1"), new SimpleCell("table02.col123.1", "dns"), new SimpleCell("table02.col124.1", "dud"), - new BinaryCell("table02.col125.1", newBlob()))), + new BinaryCell("table02.col125.1", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))), new Row(3, Arrays.asList(new SimpleCell("table02.col121.2", "3"), new SimpleCell("table02.col122.2", "2"), new SimpleCell("table02.col123.2", "usl"), new SimpleCell("table02.col124.2", "aps"), - new BinaryCell("table02.col125.2", newBlob()))))); + new BinaryCell("table02.col125.2", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))))); tableRows.put("schema02.table01", new ArrayList()); tableRows.put("schema02.table02", new ArrayList()); diff --git a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java index 89548ca6f..11ac617e7 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java +++ b/dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java @@ -15,6 +15,7 @@ import org.apache.commons.lang3.tuple.Pair; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.modules.DatabaseImportModule; import com.databasepreservation.model.modules.DatabaseModuleFactory; @@ -60,67 +61,75 @@ public DatabaseImportModule getImportModule() throws ModuleException { public void migrate() throws ModuleException { validate(); - // get import module and export module instance - Map importParameters = buildParametersFromStringParameters(importModuleFactory, - importModuleFactoryStringParameters); - Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, - exportModuleFactoryStringParameters); - - DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); - DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); - - List beforeFilterModules = new ArrayList<>(); - List afterFilterModules = new ArrayList<>(); - for (int i = 0; i < filterFactories.size(); i++) { - Map filterParameters = new HashMap<>(); - if (!filterFactoriesStringParameters.isEmpty()) { - filterParameters = buildParametersFromStringParameters(filterFactories.get(i), - filterFactoriesStringParameters.get(i)); + try { + if (exportModuleFactory != null) { + ExportModuleContextManager.getInstance().setup(exportModuleFactory); } - if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { - afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); - } else { - beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + // get import module and export module instance + Map importParameters = buildParametersFromStringParameters(importModuleFactory, + importModuleFactoryStringParameters); + Map exportParameters = buildParametersFromStringParameters(exportModuleFactory, + exportModuleFactoryStringParameters); + + DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter); + DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter); + + List beforeFilterModules = new ArrayList<>(); + List afterFilterModules = new ArrayList<>(); + for (int i = 0; i < filterFactories.size(); i++) { + Map filterParameters = new HashMap<>(); + if (!filterFactoriesStringParameters.isEmpty()) { + filterParameters = buildParametersFromStringParameters(filterFactories.get(i), + filterFactoriesStringParameters.get(i)); + } + + if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) { + afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } else { + beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter)); + } } - } - // set reporters - importModule.setOnceReporter(reporter); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - filterModule.setOnceReporter(reporter); - } + // set reporters + importModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : beforeFilterModules) { + filterModule.setOnceReporter(reporter); + } - for (DatabaseFilterModule filterModule : afterFilterModules) { - filterModule.setOnceReporter(reporter); - } - for (DatabaseFilterModule filterModule : filterModules) { - filterModule.setOnceReporter(reporter); - } - exportModule.setOnceReporter(reporter); + for (DatabaseFilterModule filterModule : afterFilterModules) { + filterModule.setOnceReporter(reporter); + } + for (DatabaseFilterModule filterModule : filterModules) { + filterModule.setOnceReporter(reporter); + } + exportModule.setOnceReporter(reporter); - // create module chain with filters in the middle - Collections.reverse(filterModules); - Collections.reverse(beforeFilterModules); - Collections.reverse(afterFilterModules); + // create module chain with filters in the middle + Collections.reverse(filterModules); + Collections.reverse(beforeFilterModules); + Collections.reverse(afterFilterModules); - DatabaseFilterModule sinkModule = new SinkModule(); + DatabaseFilterModule sinkModule = new SinkModule(); - for (DatabaseFilterModule filterModule : afterFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : afterFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - sinkModule = exportModule.migrateDatabaseTo(sinkModule); + sinkModule = exportModule.migrateDatabaseTo(sinkModule); - for (DatabaseFilterModule filterModule : beforeFilterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : beforeFilterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - for (DatabaseFilterModule filterModule : filterModules) { - sinkModule = filterModule.migrateDatabaseTo(sinkModule); - } + for (DatabaseFilterModule filterModule : filterModules) { + sinkModule = filterModule.migrateDatabaseTo(sinkModule); + } - importModule.migrateDatabaseTo(sinkModule); + importModule.migrateDatabaseTo(sinkModule); + } finally { + ExportModuleContextManager.destroy(); + } } /** diff --git a/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java new file mode 100644 index 000000000..ffe7da3ef --- /dev/null +++ b/dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java @@ -0,0 +1,43 @@ +package com.databasepreservation.managers; + +import com.databasepreservation.model.exception.UnsupportedModuleException; +import com.databasepreservation.model.modules.DatabaseModuleFactory; +import com.databasepreservation.model.parameters.Parameters; + +/** + * @author Gabriel Barros + */ +public class ExportModuleContextManager { + private static ExportModuleContextManager instance = null; + private String moduleName; + private Parameters exportModuleParameters; + + public static ExportModuleContextManager getInstance() { + if (instance == null) { + instance = new ExportModuleContextManager(); + } + + return instance; + } + + public static void destroy() { + instance = null; + } + + public void setup(DatabaseModuleFactory exportModuleFactory) throws UnsupportedModuleException { + moduleName = exportModuleFactory.getModuleName(); + exportModuleParameters = exportModuleFactory.getExportModuleParameters(); + } + + public String getModuleName() { + return moduleName; + } + + public Parameters getExportModuleParameters() { + return exportModuleParameters; + } + + public boolean isSiadDKModule() { + return moduleName.contains("siard-dk"); + } +} diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java index 8b9149f11..a5a028314 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java @@ -28,6 +28,7 @@ public class BinaryCell extends Cell implements InputStreamProvider { private InputStreamProvider inputStreamProvider; private String file; private long length; + private String mimeType; /** * Creates a binary cell. This binary cell will mostly just be a wrapper around @@ -75,6 +76,12 @@ public BinaryCell(String id, InputStreamProvider inputStreamProvider) { this.inputStreamProvider = inputStreamProvider; } + public BinaryCell(String id, InputStreamProvider inputStreamProvider, String mimeType) { + super(id); + this.inputStreamProvider = inputStreamProvider; + this.mimeType = mimeType; + } + /** * Creates a binary cell. This binary cell is a wrapper around a * ProvidesInputStream object (whilst also providing Cell functionality). @@ -128,4 +135,12 @@ public String getFile() { public long getLength() { return length; } + + public String getMimeType() { + return mimeType; + } + + public void setFile(String file) { + this.file = file; + } } diff --git a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java index 03f74e006..378250821 100644 --- a/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java +++ b/dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java @@ -28,7 +28,7 @@ public enum INPUT_TYPE { /* GUI Helper for SIARD Export Module */ public enum CATEGORY_TYPE { - SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, NONE + SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, CONVERSION_SERVICE_OPTIONS, NONE } public enum FILE_FILTER_TYPE { @@ -312,7 +312,9 @@ public Parameter defaultSelectedIndex(Integer index) { return this; } - public Integer getDefaultSelectedIndex() { return defaultSelectedIndex; } + public Integer getDefaultSelectedIndex() { + return defaultSelectedIndex; + } /** * Gets the export option type for this parameter; Helper to automatize the diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java index 162692682..2c59e4474 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java @@ -40,7 +40,9 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { return newCell; } - Path blobPath = basePath.resolve(cellValue); + String cellValueStripTrailing = cellValue.stripTrailing(); + + Path blobPath = basePath.resolve(cellValueStripTrailing); if (Files.exists(blobPath)) { if (Files.isRegularFile(blobPath)) { @@ -48,15 +50,15 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException { newCell = new BinaryCell(cellId, new PathInputStreamProvider(blobPath)); } catch (ModuleException e) { reporter.ignored("Cell " + cellId, - blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, - blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValue); + blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing); } } else { reporter.ignored("Cell " + cellId, "Path: " + blobPath.toString() + " could not be found; Base path: " - + this.basePath + " Cell Value: " + cellValue); + + this.basePath + " Cell Value: " + cellValueStripTrailing); } return newCell; } diff --git a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java index bd34df3ab..b64d47e6a 100644 --- a/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java +++ b/dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java @@ -14,10 +14,13 @@ import java.util.Map; import java.util.Set; +import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.managers.ExportModuleContextManager; import com.databasepreservation.managers.ModuleConfigurationManager; +import com.databasepreservation.model.data.BinaryCell; import com.databasepreservation.model.data.Cell; import com.databasepreservation.model.data.NullCell; import com.databasepreservation.model.data.Row; @@ -93,12 +96,7 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException Type original = column.getType(); description.append(". Original description: '").append(original.getDescription()).append("')"); - SimpleTypeBinary newType = new SimpleTypeBinary(); - newType.setSql99TypeName("BINARY VARYING", 1); - newType.setSql2008TypeName("BINARY VARYING", 1); - newType.setOriginalTypeName(original.getOriginalTypeName()); - newType.setOutsideDatabase(true); - + SimpleTypeBinary newType = getSimpleTypeBinary(original); column.setType(newType); column.setDescription(description.toString()); } @@ -110,6 +108,21 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException this.exportModule.handleStructure(structure); } + @NotNull + private static SimpleTypeBinary getSimpleTypeBinary(Type original) { + SimpleTypeBinary newType = new SimpleTypeBinary(); + if (ExportModuleContextManager.getInstance().isSiadDKModule()) { + newType.setSql99TypeName("BINARY LARGE OBJECT"); + newType.setSql2008TypeName("BINARY LARGE OBJECT"); + } else { + newType.setSql99TypeName("BINARY VARYING", 1); + newType.setSql2008TypeName("BINARY VARYING", 1); + } + newType.setOriginalTypeName(original.getOriginalTypeName()); + newType.setOutsideDatabase(true); + return newType; + } + @Override public void handleDataOpenSchema(String schemaName) throws ModuleException { this.exportModule.handleDataOpenSchema(schemaName); @@ -155,6 +168,9 @@ public void handleDataRow(Row row) throws ModuleException { .get(currentTable.getId() + index); Cell newCell = getExternalLOBSCellHandler(externalLobsConfiguration).handleCell(cell.getId(), simpleCell.getSimpleData()); + if (newCell instanceof BinaryCell binaryCell) { + binaryCell.setFile(simpleCell.getSimpleData()); + } rowCells.set(index, newCell); } else { reporter.ignored("Cell " + cell.getId(), "reference to external LOB is null"); diff --git a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java index 9d029d73c..92a7b009e 100644 --- a/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java +++ b/dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java @@ -684,7 +684,7 @@ protected void handleSimpleTypeStringDataCell(String data, PreparedStatement ps, protected void handleSimpleTypeNumericExactDataCell(String data, PreparedStatement ps, int index, Cell cell, ColumnStructure column) throws SQLException { - if (data != null) { + if (data != null && !data.isEmpty()) { BigDecimal bd = new BigDecimal(data); ps.setBigDecimal(index, bd); } else { diff --git a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java index 7d63862a6..de1449ad1 100644 --- a/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java +++ b/dbptk-modules/dbptk-module-postgresql/src/main/java/com/databasepreservation/modules/postgresql/PostgreSQLHelper.java @@ -59,7 +59,7 @@ public String escapeSchemaName(String schema) { @Override public String escapeTableName(String table) { - return getStartQuote() + table + getEndQuote(); + return getStartQuote() + StringUtils.strip(table, "\"") + getEndQuote(); } /** diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java index f1bc0850a..87e038dcd 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/SIARDDKModuleFactory.java @@ -42,6 +42,9 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { public static final String PARAMETER_AS_SCHEMA = "as-schema"; public static final String PARAMETER_LOBS_PER_FOLDER = "lobs-per-folder"; public static final String PARAMETER_LOBS_FOLDER_SIZE = "lobs-folder-size"; + public static final String PARAMETER_LOB_CONVERSION_ENABLED = "lob-conversion"; + public static final String PARAMETER_LOB_CONVERSION_ENDPOINT = "lob-conversion-endpoint"; + public static final String PARAMETER_LOB_CONVERSION_TARGET_FORMAT = "lob-conversion-target-format"; // TODO: As things are now, are we not always generating the '.1' version of // the archive (indicating that the last .[1-9][0-9] should perhaps not be @@ -75,6 +78,20 @@ public abstract class SIARDDKModuleFactory implements DatabaseModuleFactory { .description("The maximum size (in megabytes) of the docCollection folders (default is 1000 MB").required(false) .hasArgument(true).setOptionalArgument(false).valueIfNotSet("1000"); + private static final Parameter lobConversionEnabled = new Parameter().shortName("lc") + .longName(PARAMETER_LOB_CONVERSION_ENABLED).description("Enables asynchronous LOB conversion via HTTP.") + .hasArgument(false).setOptionalArgument(false).required(false).valueIfSet("true").valueIfNotSet("false"); + + private static final Parameter lobConversionEndpoint = new Parameter().shortName("lce") + .longName(PARAMETER_LOB_CONVERSION_ENDPOINT) + .description("The API endpoint URL for the LOB conversion service (default is http://localhost:8087).") + .hasArgument(true).setOptionalArgument(false).required(false).valueIfNotSet("http://localhost:8087"); + + private static final Parameter lobConversionTargetFormat = new Parameter().shortName("lcf") + .longName(PARAMETER_LOB_CONVERSION_TARGET_FORMAT) + .description("Target MIME type format for the LOB conversion (default is image/tiff).").hasArgument(true) + .setOptionalArgument(false).required(false).valueIfNotSet("image/tiff"); + // This is not used now, but will be used later // private static final Parameter clobType = new // Parameter().shortName("ct").longName("clobtype") @@ -120,6 +137,9 @@ public Map getAllParameters() { parameterMap.put(importAsSchema.longName(), importAsSchema); parameterMap.put(lobsPerFolder.longName(), lobsPerFolder); parameterMap.put(lobsFolderSize.longName(), lobsFolderSize); + parameterMap.put(lobConversionEnabled.longName(), lobConversionEnabled); + parameterMap.put(lobConversionEndpoint.longName(), lobConversionEndpoint); + parameterMap.put(lobConversionTargetFormat.longName(), lobConversionTargetFormat); // to be used later... // parameterMap.put(clobType.longName(), clobType); // parameterMap.put(clobLength.longName(), clobLength); @@ -145,18 +165,22 @@ public Parameters getExportModuleParameters() throws UnsupportedModuleException // contextDocumentationIndex, contextDocmentationFolder, // clobType, clobLength), null); - return new Parameters( - Arrays.asList( - folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) - .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) - .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), - lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), - lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS)), + return new Parameters(Arrays.asList( + folder.inputType(Parameter.INPUT_TYPE.FOLDER).exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + archiveIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN).fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationIndex.inputType(Parameter.INPUT_TYPE.FILE_OPEN) + .fileFilter(Parameter.FILE_FILTER_TYPE.XML_EXTENSION) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + contextDocumentationFolder.inputType(Parameter.INPUT_TYPE.FOLDER) + .exportOptions(Parameter.CATEGORY_TYPE.SIARD_EXPORT_OPTIONS), + lobsPerFolder.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobsFolderSize.inputType(Parameter.INPUT_TYPE.TEXT).exportOptions(Parameter.CATEGORY_TYPE.EXTERNAL_LOBS), + lobConversionEnabled.inputType(Parameter.INPUT_TYPE.CHECKBOX).exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), + lobConversionEndpoint.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS), + lobConversionTargetFormat.inputType(Parameter.INPUT_TYPE.TEXT) + .exportOptions(Parameter.CATEGORY_TYPE.CONVERSION_SERVICE_OPTIONS)), null); } @@ -193,6 +217,21 @@ public DatabaseFilterModule buildExportModule(Map parameters, pLobsFolderSize = parameters.get(lobsFolderSize); } + String pLobConversionEnabled = lobConversionEnabled.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEnabled))) { + pLobConversionEnabled = parameters.get(lobConversionEnabled); + } + + String pLobConversionEndpoint = lobConversionEndpoint.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionEndpoint))) { + pLobConversionEndpoint = parameters.get(lobConversionEndpoint); + } + + String pLobConversionTargetFormat = lobConversionTargetFormat.valueIfNotSet(); + if (StringUtils.isNotBlank(parameters.get(lobConversionTargetFormat))) { + pLobConversionTargetFormat = parameters.get(lobConversionTargetFormat); + } + // to be used later... // String pClobType = parameters.get(clobType); // String pClobLength = parameters.get(clobLength); @@ -205,6 +244,9 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleArgs.put(contextDocumentationFolder.longName(), pContextDocumentationFolder); exportModuleArgs.put(lobsPerFolder.longName(), pLobsPerFolder); exportModuleArgs.put(lobsFolderSize.longName(), pLobsFolderSize); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENABLED, pLobConversionEnabled); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_ENDPOINT, pLobConversionEndpoint); + exportModuleArgs.put(PARAMETER_LOB_CONVERSION_TARGET_FORMAT, pLobConversionTargetFormat); // to be used later... // exportModuleArgs.put(clobType.longName(), pClobType); @@ -227,6 +269,18 @@ public DatabaseFilterModule buildExportModule(Map parameters, exportModuleParameters.add(lobsFolderSize.longName()); exportModuleParameters.add(pLobsFolderSize); } + if (!pLobConversionEnabled.equals(lobConversionEnabled.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENABLED); + exportModuleParameters.add(pLobConversionEnabled); + } + if (!pLobConversionEndpoint.equals(lobConversionEndpoint.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_ENDPOINT); + exportModuleParameters.add(pLobConversionEndpoint); + } + if (!pLobConversionTargetFormat.equals(lobConversionTargetFormat.valueIfNotSet())) { + exportModuleParameters.add(PARAMETER_LOB_CONVERSION_TARGET_FORMAT); + exportModuleParameters.add(pLobConversionTargetFormat); + } reporter.exportModuleParameters(getModuleName(), exportModuleParameters.toArray(new String[0])); return createSIARDDKExportModuleInstance(exportModuleArgs).getDatabaseExportModule(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java index a4fe44a1a..f0a445522 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/common/path/SIARDDKMetadataPathStrategy.java @@ -7,12 +7,12 @@ */ package com.databasepreservation.modules.siard.common.path; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; - import java.security.InvalidParameterException; import java.util.ArrayList; import java.util.Arrays; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + /** * @author Andreas Kring * @@ -54,7 +54,8 @@ public boolean checkFilename(String filename) { // Valid filenames String[] validFileNames = {SIARDDKConstants.TABLE_INDEX, SIARDDKConstants.ARCHIVE_INDEX, SIARDDKConstants.DOC_INDEX, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, SIARDDKConstants.FILE_INDEX, - SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, "fileIndex_original", "docIndex_original"}; + SIARDDKConstants.DOCUMENT_IDENTIFICATION, SIARDDKConstants.XML_SCHEMA, SIARDDKConstants.RESEARCH_INDEX, + "fileIndex_original", "docIndex_original"}; ArrayList validFilenames = new ArrayList(Arrays.asList(validFileNames)); if (validFilenames.contains(filename)) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java index 68dd9d90b..203433413 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/constants/SIARDDKConstants.java @@ -54,6 +54,7 @@ public class SIARDDKConstants { public static final String ARCHIVE_INDEX = "archiveIndex"; public static final String TABLE_INDEX = "tableIndex"; public static final String FILE_INDEX = "fileIndex"; + public static final String RESEARCH_INDEX = "researchIndex"; public static final String DOC_INDEX = "docIndex"; public static final String DOCUMENT_IDENTIFICATION = "documentIdentification"; public static final String XML_SCHEMA = "XMLSchema"; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java index 8f94967f8..4f2292090 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007ContentImportStrategy.java @@ -32,7 +32,6 @@ import dk.sa.xmlns.diark._1_0.docindex.DocIndexType; import dk.sa.xmlns.diark._1_0.docindex.DocumentType; import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Unmarshaller; @@ -48,56 +47,9 @@ public SIARDDK1007ContentImportStrategy(FolderReadStrategyMD5Sum readStrategy, S super(readStrategy, pathStrategy, importAsSchema); } - DocIndexType loadVirtualTableContent() throws ModuleException, FileNotFoundException { - JAXBContext context; - try { - context = JAXBContext.newInstance(DocIndexType.class.getPackage().getName()); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error loading JAXBContext").withCause(e); - } - - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema xsdSchema = null; - InputStream xsdInputStream = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)); - - try { - xsdSchema = schemaFactory.newSchema(new StreamSource(xsdInputStream)); - } catch (SAXException e) { - throw new ModuleException() - .withMessage("Error reading metadata XSD file: " + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)) - .withCause(e); - } - InputStream inputStreamXml = null; - Unmarshaller unmarshaller; - try { - unmarshaller = context.createUnmarshaller(); - unmarshaller.setSchema(xsdSchema); - inputStreamXml = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX)); - Object result = unmarshaller.unmarshal(inputStreamXml); - DocIndexType docIndex; - if (result instanceof JAXBElement) { - docIndex = ((JAXBElement) result).getValue(); - } else if (result instanceof DocIndexType) { - docIndex = (DocIndexType) result; - } else { - throw new IllegalArgumentException("Unexpected object type: " + result.getClass().getName()); - } - return docIndex; - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error while Unmarshalling JAXB").withCause(e); - } finally { - try { - xsdInputStream.close(); - if (inputStreamXml != null) { - inputStreamXml.close(); - xsdInputStream.close(); - } - } catch (IOException e) { - logger.debug("Could not close xsdStream", e); - } - } + @Override + SIARDDKDocIndexHandler createDocIndexContentHandler(Schema xsdSchema) { + return new SIARDDK1007DocIndexHandler(pathStrategy, dbExportHandler); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java new file mode 100644 index 000000000..e50026fa7 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK1007DocIndexHandler.java @@ -0,0 +1,56 @@ +package com.databasepreservation.modules.siard.in.content; + +import com.databasepreservation.model.modules.DatabaseExportModule; +import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK1007DocIndexHandler extends SIARDDKDocIndexHandler { + + public SIARDDK1007DocIndexHandler(SIARDDKPathImportStrategy pathImportStrategy, + DatabaseExportModule databaseExportModule) { + super(pathImportStrategy, databaseExportModule); + } + + @Override + String getDocLocalName() { + return "doc"; + } + + @Override + String getDocIDLocalName() { + return "dID"; + } + + @Override + String getParentIDLocalName() { + return "pID"; + } + + @Override + String getMediaIDLocalName() { + return "mID"; + } + + @Override + String getContainerFolderLocalName() { + return "dCf"; + } + + @Override + String getOriginalFilenameLocalName() { + return "oFn"; + } + + @Override + String getArchivalFileTypeLocalName() { + return "aFt"; + } + + @Override + String getGmlXSDLocalName() { + return "gmlXsd"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java index fc376901c..85550112d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128ContentImportStrategy.java @@ -24,15 +24,14 @@ import org.xml.sax.SAXException; import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.bindings.siard_dk_128.context.ContextDocumentationIndex; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocumentType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.context.ContextDocumentationIndex; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; import com.databasepreservation.modules.siard.in.read.FolderReadStrategyMD5Sum; import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBElement; import jakarta.xml.bind.JAXBException; import jakarta.xml.bind.Unmarshaller; @@ -54,56 +53,9 @@ public SIARDDK128ContentImportStrategy(FolderReadStrategyMD5Sum readStrategy, SI super(readStrategy, pathStrategy, importAsSchema); } - DocIndexType loadVirtualTableContent() throws ModuleException, FileNotFoundException { - JAXBContext context; - try { - context = JAXBContext.newInstance(DocIndexType.class.getPackage().getName()); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error loading JAXBContext").withCause(e); - } - - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema xsdSchema = null; - InputStream xsdInputStream = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)); - - try { - xsdSchema = schemaFactory.newSchema(new StreamSource(xsdInputStream)); - } catch (SAXException e) { - throw new ModuleException() - .withMessage("Error reading metadata XSD file: " + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX)) - .withCause(e); - } - InputStream inputStreamXml = null; - Unmarshaller unmarshaller; - try { - unmarshaller = context.createUnmarshaller(); - unmarshaller.setSchema(xsdSchema); - inputStreamXml = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() - + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX)); - Object result = unmarshaller.unmarshal(inputStreamXml); - DocIndexType docIndex; - if (result instanceof JAXBElement) { - docIndex = ((JAXBElement) result).getValue(); - } else if (result instanceof DocIndexType) { - docIndex = (DocIndexType) result; - } else { - throw new IllegalArgumentException("Unexpected object type: " + result.getClass().getName()); - } - return docIndex; - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error while Unmarshalling JAXB").withCause(e); - } finally { - try { - xsdInputStream.close(); - if (inputStreamXml != null) { - inputStreamXml.close(); - xsdInputStream.close(); - } - } catch (IOException e) { - logger.debug("Could not close xsdStream", e); - } - } + @Override + SIARDDKDocIndexHandler createDocIndexContentHandler(Schema xsdSchema) { + return new SIARDDK128DocIndexHandler(pathStrategy, dbExportHandler); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java new file mode 100644 index 000000000..cfaae5d25 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDK128DocIndexHandler.java @@ -0,0 +1,56 @@ +package com.databasepreservation.modules.siard.in.content; + +import com.databasepreservation.model.modules.DatabaseExportModule; +import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK128DocIndexHandler extends SIARDDKDocIndexHandler { + + public SIARDDK128DocIndexHandler(SIARDDKPathImportStrategy pathImportStrategy, + DatabaseExportModule databaseExportModule) { + super(pathImportStrategy, databaseExportModule); + } + + @Override + String getDocLocalName() { + return "doc"; + } + + @Override + String getDocIDLocalName() { + return "dID"; + } + + @Override + String getParentIDLocalName() { + return "pID"; + } + + @Override + String getMediaIDLocalName() { + return "mID"; + } + + @Override + String getContainerFolderLocalName() { + return "dCf"; + } + + @Override + String getOriginalFilenameLocalName() { + return "oFn"; + } + + @Override + String getArchivalFileTypeLocalName() { + return "aFt"; + } + + @Override + String getGmlXSDLocalName() { + return "gmlXsd"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java index 011c33e46..eefe3da8b 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKContentImportStrategy.java @@ -8,8 +8,10 @@ package com.databasepreservation.modules.siard.in.content; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; +import java.io.InputStream; import java.math.BigInteger; import java.nio.file.Files; import java.nio.file.Path; @@ -299,70 +301,47 @@ void populateContextDocumentationTable(TableStructure table) throws ModuleExcept } void populateVirtualTable(TableStructure table) throws ModuleException, FileNotFoundException { - D docIndex = loadVirtualTableContent(); - currentTable = table; this.dbExportHandler.handleDataOpenTable(table.getId()); - int rowCounter = 0; - for (T doc : getDocuments(docIndex)) { - Row row = new Row(); - List lstCells = new ArrayList<>(); - row.setIndex(rowCounter); - - // document id - Cell dIDCell = new SimpleCell(SIARDDKConstants.DID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, - getDID(doc).toString()); - lstCells.add(dIDCell); - - // parent id - BigInteger pID = getPID(doc); - String pIDString = pID == null ? "" : pID.toString(); - - Cell pIDCell = new SimpleCell(SIARDDKConstants.PID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, - pIDString); - lstCells.add(pIDCell); - - try { - // document blob - String mainFolder = pathStrategy.getMainFolder().getPath().toString(); - String siardFolderName = mainFolder.substring(0, mainFolder.length() - 1) + getMID(doc); - Path siardFolderPath = Paths.get(siardFolderName); - Path docPath = siardFolderPath - .resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME, getDCf(doc), getDID(doc).toString())); - - if (!docPath.startsWith(siardFolderPath.resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME)))) { - throw new ModuleException().withMessage("Invalid path for folder: " + docPath); - } - - String digest = ""; - File docFolder = new File(docPath.toString()); - if (docFolder.exists() && docFolder.isDirectory()) { - File[] fileList = docFolder.listFiles(); - if (fileList != null && fileList.length == 1) { - docPath = docPath.resolve(Paths.get(fileList[0].getName())); - digest = DigestUtils.sha1Hex(Files.newInputStream(docPath)); - } - } - - Cell blobCell = new BinaryCell( - SIARDDKConstants.BLOB_EXTENSION + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, - new DummyInputStreamProvider(), docPath.toString(), Files.size(docPath), digest, - DigestUtils.getSha1Digest().toString()); - lstCells.add(blobCell); + currentTable = table; + loadVirtualTableContent(); + this.dbExportHandler.handleDataCloseTable(table.getId()); + } - // set and handle row - assert !lstCells.contains(null); - row.setCells(lstCells); - this.dbExportHandler.handleDataRow(row); + void loadVirtualTableContent() throws ModuleException, FileNotFoundException { + ValidatorHandler validatorHandler; + + try (InputStream xsdStream = new FileInputStream(pathStrategy.getMainFolder().getPath().toString() + + SIARDDKConstants.RESOURCE_FILE_SEPARATOR + pathStrategy.getXsdFilePath(SIARDDKConstants.DOC_INDEX))){ + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema xsdSchema = schemaFactory.newSchema(new StreamSource(xsdStream)); + SIARDDKDocIndexHandler docIndexHandler = createDocIndexContentHandler(xsdSchema); + validatorHandler = createDocIndexValidatorHandler(xsdSchema, docIndexHandler); + } catch (IOException | SAXException e) { + throw new ModuleException().withMessage("Error while preparing doc index parser").withCause(e); + } - rowCounter++; - } catch (ModuleException | IOException e) { - throw new ModuleException().withMessage("Error handling data row index:" + rowCounter).withCause(e); - } + try { + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setValidating(false); + saxParserFactory.setNamespaceAware(true); + SAXParser saxParser = saxParserFactory.newSAXParser(); + XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setContentHandler(validatorHandler); + xmlReader.parse(new InputSource(readStrategy.createInputStream(pathStrategy.getMainFolder(), + pathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX)))); + } catch (SAXException | ParserConfigurationException | IOException e) { + throw new ModuleException().withMessage("Error while parsing doc index").withCause(e); } - this.dbExportHandler.handleDataCloseTable(table.getId()); } - abstract D loadVirtualTableContent() throws ModuleException, FileNotFoundException; + ValidatorHandler createDocIndexValidatorHandler(Schema xsdSchema, SIARDDKDocIndexHandler docIndexHandler) { + ValidatorHandler validatorHandler = xsdSchema.newValidatorHandler(); + validatorHandler.setContentHandler(docIndexHandler); + return validatorHandler; + } + + abstract SIARDDKDocIndexHandler createDocIndexContentHandler(Schema xsdSchema); + abstract F loadContextDocTableContent() throws ModuleException, FileNotFoundException; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java new file mode 100644 index 000000000..7deed4963 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexDoc.java @@ -0,0 +1,80 @@ +package com.databasepreservation.modules.siard.in.content; + +import java.io.Serializable; +import java.math.BigInteger; + +/** + * + * @author Alexandre Flores + */ + +public class SIARDDKDocIndexDoc implements Serializable { + BigInteger documentID; + BigInteger parentID; + BigInteger mediaID; + String documentCollectionFolder; + String originalFilename; + String archivalFileType; + String gmlXSD; + + public SIARDDKDocIndexDoc() { + + } + + public BigInteger getParentID() { + return parentID; + } + + public void setParentID(BigInteger parentID) { + this.parentID = parentID; + } + + public BigInteger getDocumentID() { + return documentID; + } + + public void setDocID(BigInteger documentID) { + this.documentID = documentID; + } + + public BigInteger getMediaID() { + return mediaID; + } + + public void setMediaID(BigInteger mediaID) { + this.mediaID = mediaID; + } + + public String getDocumentCollectionFolder() { + return documentCollectionFolder; + } + + public void setContainerFolder(String documentCollectionFolder) { + this.documentCollectionFolder = documentCollectionFolder; + } + + public String getOriginalFilename() { + return originalFilename; + } + + public void setOriginalFilename(String originalFilename) { + this.originalFilename = originalFilename; + } + + public String getArchivalFileType() { + return archivalFileType; + } + + public void setArchivalFileType(String archivalFileType) { + this.archivalFileType = archivalFileType; + } + + public String getGmlXSD() { + return gmlXSD; + } + + public void setGmlXSD(String gmlXSD) { + this.gmlXSD = gmlXSD; + } + +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java new file mode 100644 index 000000000..3c9028805 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/content/SIARDDKDocIndexHandler.java @@ -0,0 +1,202 @@ +package com.databasepreservation.modules.siard.in.content; + +import java.io.File; +import java.io.IOException; +import java.math.BigInteger; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import com.databasepreservation.modules.siard.in.path.SIARDDKPathImportStrategy; +import org.apache.commons.codec.digest.DigestUtils; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import com.databasepreservation.common.io.providers.DummyInputStreamProvider; +import com.databasepreservation.model.data.BinaryCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.model.data.Row; +import com.databasepreservation.model.data.SimpleCell; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.modules.DatabaseExportModule; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + +/** + * + * @author Alexandre Flores + */ +public abstract class SIARDDKDocIndexHandler extends DefaultHandler { + protected SIARDDKDocIndexDoc currentDoc; + protected StringBuilder currentElementCharacters; + + private SIARDDKPathImportStrategy pathImportStrategy; + private DatabaseExportModule databaseExportModule; + private int rowCounter; + + public SIARDDKDocIndexHandler(SIARDDKPathImportStrategy pathImportStrategy, + DatabaseExportModule databaseExportModule) { + this.pathImportStrategy = pathImportStrategy; + this.databaseExportModule = databaseExportModule; + rowCounter = 0; + } + + @Override + public void startDocument() throws SAXException { + // no op + } + + @Override + public void endDocument() throws SAXException { + // no op + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + currentElementCharacters = new StringBuilder(); + + if (localName.equals(getDocLocalName())) { + startElementDoc(uri, localName, qName, attributes); + } + } + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + if (localName.equals(getDocLocalName())) { + endElementDoc(); + } + else if (localName.equals(getDocIDLocalName())) { + endElementDocID(); + } + else if (localName.equals(getContainerFolderLocalName())) { + endElementContainerFolder(); + } + else if (localName.equals(getGmlXSDLocalName())) { + endElementGmlXSD(); + } + else if (localName.equals(getMediaIDLocalName())) { + endElementMediaID(); + } + else if (localName.equals(getArchivalFileTypeLocalName())) { + endElementArchivalFileType(); + } + else if (localName.equals(getParentIDLocalName())) { + endElementParentID(); + } + else if (localName.equals(getOriginalFilenameLocalName())) { + endElementOriginalFilename(); + } + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + currentElementCharacters.append(ch, start, length); + } + + protected void startElementDoc(String uri, String localName, String qName, Attributes attributes) { + this.currentDoc = new SIARDDKDocIndexDoc(); + } + + protected void endElementDoc() throws SAXException { + Row row = new Row(); + List lstCells = new ArrayList<>(); + row.setIndex(rowCounter); + + // document id + Cell dIDCell = new SimpleCell(SIARDDKConstants.DID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, + currentDoc.getDocumentID().toString()); + lstCells.add(dIDCell); + + // parent id + BigInteger pID = currentDoc.getParentID(); + String pIDString = pID == null ? "" : pID.toString(); + + Cell pIDCell = new SimpleCell(SIARDDKConstants.PID + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, + pIDString); + lstCells.add(pIDCell); + + try { + // document blob + String mainFolder = pathImportStrategy.getMainFolder().getPath().toString(); + String siardFolderName = mainFolder.substring(0, mainFolder.length() - 1) + currentDoc.getMediaID(); + Path siardFolderPath = Paths.get(siardFolderName); + Path docPath = siardFolderPath.resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME, + currentDoc.getDocumentCollectionFolder(), currentDoc.getDocumentID().toString())); + + if (!docPath.startsWith(siardFolderPath.resolve(Paths.get(SIARDDKConstants.DOCUMENTS_FOLDER_NAME)))) { + throw new ModuleException().withMessage("Invalid path for folder: " + docPath); + } + + String digest = ""; + File docFolder = new File(docPath.toString()); + if (docFolder.exists() && docFolder.isDirectory()) { + File[] fileList = docFolder.listFiles(); + if (fileList != null && fileList.length == 1) { + docPath = docPath.resolve(Paths.get(fileList[0].getName())); + digest = DigestUtils.sha1Hex(Files.newInputStream(docPath)); + } + } + + Cell blobCell = new BinaryCell( + SIARDDKConstants.BLOB_EXTENSION + SIARDDKConstants.FILE_EXTENSION_SEPARATOR + rowCounter, + new DummyInputStreamProvider(), docPath.toString(), Files.size(docPath), digest, + DigestUtils.getSha1Digest().toString()); + lstCells.add(blobCell); + + // set and handle row + assert !lstCells.contains(null); + row.setCells(lstCells); + this.databaseExportModule.handleDataRow(row); + + rowCounter++; + } catch (ModuleException | IOException e) { + throw new SAXException("Error handling data row index:" + rowCounter, e); + } + } + + protected void endElementDocID() { + this.currentDoc.setDocID(new BigInteger(this.currentElementCharacters.toString())); + } + + protected void endElementParentID() { + this.currentDoc.setParentID(new BigInteger(this.currentElementCharacters.toString())); + } + + protected void endElementMediaID() { + this.currentDoc.setMediaID(new BigInteger(this.currentElementCharacters.toString())); + } + + protected void endElementContainerFolder() { + this.currentDoc.setContainerFolder(this.currentElementCharacters.toString()); + } + + protected void endElementOriginalFilename() { + this.currentDoc.setOriginalFilename(this.currentElementCharacters.toString()); + } + + protected void endElementArchivalFileType() { + this.currentDoc.setArchivalFileType(this.currentElementCharacters.toString()); + } + + protected void endElementGmlXSD() { + this.currentDoc.setGmlXSD(this.currentElementCharacters.toString()); + } + + abstract String getDocLocalName(); + + abstract String getDocIDLocalName(); + + abstract String getParentIDLocalName(); + + abstract String getMediaIDLocalName(); + + abstract String getContainerFolderLocalName(); + + abstract String getOriginalFilenameLocalName(); + + abstract String getArchivalFileTypeLocalName(); + + abstract String getGmlXSDLocalName(); +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java index 8af36d970..6e1e660bb 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007ExtMetadataImportStrategy.java @@ -143,7 +143,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java index e1d7289a9..682c6e136 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK1007MetadataImportStrategy.java @@ -142,7 +142,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java index 16ee2857d..10dd689da 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128ExtMetadataImportStrategy.java @@ -142,7 +142,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java index b50faefab..25bdeb1fe 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/metadata/SIARDDK128MetadataImportStrategy.java @@ -142,7 +142,7 @@ public void loadMetadata(ReadStrategy readStrategy, SIARDArchiveContainer contai tableIndexUnmarshaller = tableIndexContext.createUnmarshaller(); tableIndexUnmarshaller.setSchema(tableIndexXsdSchema); tableIndexInputStreamXml = readStrategyMD5Sum.createInputStream(container, - pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTabelIndexExpectedMD5Sum()); + pathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX), pathStrategy.getTableIndexExpectedMD5Sum()); xmlRoot = (SiardDiark) tableIndexUnmarshaller.unmarshal(tableIndexInputStreamXml); archiveIndexUnmarshaller = archiveIndexContext.createUnmarshaller(); @@ -323,7 +323,7 @@ private TableStructure createContextDocumentationTable() throws ModuleException virtualTable.setRows(contextDocumentationIndex.getDocument().size()); virtualTable.setColumns(createContextDocumentsTableColumns()); virtualTable.setPrimaryKey(createVirtualPrimaryKey( - SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DID)); + SIARDDKConstants.CONTEXT_DOCUMENTATION_VIRTUAL_TABLE_PRIMARY_KEY_NAME, SIARDDKConstants.DOCUMENT_ID)); return virtualTable; } } catch (FileNotFoundException e) { diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java index acd8386f6..f6d341df8 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007ExtPathImportStrategy.java @@ -32,8 +32,9 @@ public SIARDDK1007ExtPathImportStrategy(SIARDArchiveContainer mainFolder, ReadSt } @Override - byte[] getMd5(F fileInfo) { - return fileInfo.getMd5(); + SIARDDKFileIndexHandler createFileIndexHandler() { + return new SIARDDK1007FileIndexHandler(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, + xmlFilePathLookupByFolderName); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java new file mode 100644 index 000000000..e46064ec3 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007FileIndexHandler.java @@ -0,0 +1,35 @@ +package com.databasepreservation.modules.siard.in.path; + +import java.util.Map; + +/** + * @author Alexandre Flores + */ +public class SIARDDK1007FileIndexHandler extends SIARDDKFileIndexHandler { + + public SIARDDK1007FileIndexHandler(Map archiveFolderLookupByFolderName, + Map xsdFilePathLookupByFolderName, + Map xmlFilePathLookupByFolderName) { + super(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, xmlFilePathLookupByFolderName); + } + + @Override + String getFileLocalName() { + return "f"; + } + + @Override + String getFileNameLocalName() { + return "fiN"; + } + + @Override + String getFolderNameLocalName() { + return "foN"; + } + + @Override + String getMD5LocalName() { + return "md5"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java index 20969bcd7..7c493c5e1 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK1007PathImportStrategy.java @@ -8,14 +8,15 @@ package com.databasepreservation.modules.siard.in.path; import java.util.List; -import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType; -import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType.F; -import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; -import com.databasepreservation.modules.siard.in.read.ReadStrategy; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType; +import com.databasepreservation.modules.siard.bindings.siard_dk_1007.FileIndexType.F; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.in.read.ReadStrategy; /** * @author António Lindo @@ -23,13 +24,17 @@ public class SIARDDK1007PathImportStrategy extends SIARDDKPathImportStrategy { protected final Logger logger = LoggerFactory.getLogger(ContentPathImportStrategy.class); - public SIARDDK1007PathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy readStrategy, MetadataPathStrategy metadataPathStrategy, String importAsSchema, FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy) { - super(mainFolder, readStrategy, metadataPathStrategy, importAsSchema, fileIndexXsdInputStreamStrategy, FileIndexType.class); + public SIARDDK1007PathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy readStrategy, + MetadataPathStrategy metadataPathStrategy, String importAsSchema, + FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy) { + super(mainFolder, readStrategy, metadataPathStrategy, importAsSchema, fileIndexXsdInputStreamStrategy, + FileIndexType.class); } @Override - byte[] getMd5(F fileInfo) { - return fileInfo.getMd5(); + SIARDDKFileIndexHandler createFileIndexHandler() { + return new SIARDDK1007FileIndexHandler(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, + xmlFilePathLookupByFolderName); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java index 7eab0d96b..1c5f92850 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128ExtPathImportStrategy.java @@ -28,8 +28,9 @@ public SIARDDK128ExtPathImportStrategy(SIARDArchiveContainer mainFolder, ReadStr } @Override - byte[] getMd5(F fileInfo) { - return fileInfo.getMd5(); + SIARDDKFileIndexHandler createFileIndexHandler() { + return new SIARDDK128FileIndexHandler(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, + xmlFilePathLookupByFolderName); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java new file mode 100644 index 000000000..64622b23e --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128FileIndexHandler.java @@ -0,0 +1,35 @@ +package com.databasepreservation.modules.siard.in.path; + +import java.util.Map; + +/** + * @author Alexandre Flores + */ +public class SIARDDK128FileIndexHandler extends SIARDDKFileIndexHandler { + + public SIARDDK128FileIndexHandler(Map archiveFolderLookupByFolderName, + Map xsdFilePathLookupByFolderName, + Map xmlFilePathLookupByFolderName) { + super(archiveFolderLookupByFolderName, xsdFilePathLookupByFolderName, xmlFilePathLookupByFolderName); + } + + @Override + String getFileLocalName() { + return "f"; + } + + @Override + String getFileNameLocalName() { + return "fiN"; + } + + @Override + String getFolderNameLocalName() { + return "foN"; + } + + @Override + String getMD5LocalName() { + return "md5"; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java index 7f768485c..0aeeb28ba 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDK128PathImportStrategy.java @@ -7,14 +7,14 @@ */ package com.databasepreservation.modules.siard.in.path; +import java.util.List; + import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType.F; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; import com.databasepreservation.modules.siard.in.read.ReadStrategy; -import java.util.List; - /** * @author António Lindo */ @@ -23,12 +23,14 @@ public class SIARDDK128PathImportStrategy extends SIARDDKPathImportStrategy + */ + +public class SIARDDKFileIndexFile implements Serializable { + String folderName; + String fileName; + byte[] md5; + + public SIARDDKFileIndexFile() { + + } + + public String getFolderName() { + return folderName; + } + + public void setFolderName(String folderName) { + this.folderName = folderName; + } + + public String getFileName() { + return fileName; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + + public byte[] getMd5() { + return md5; + } + + public void setMd5(byte[] md5) { + this.md5 = md5; + } + +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java new file mode 100644 index 000000000..ef82fe522 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKFileIndexHandler.java @@ -0,0 +1,153 @@ +package com.databasepreservation.modules.siard.in.path; + +import java.nio.file.FileSystems; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.commons.codec.DecoderException; +import org.apache.commons.codec.binary.Hex; +import org.xml.sax.Attributes; +import org.xml.sax.SAXException; +import org.xml.sax.helpers.DefaultHandler; + +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; + +/** + * + * @author Alexandre Flores + */ +public abstract class SIARDDKFileIndexHandler extends DefaultHandler { + + protected final Map archiveFolderLookupByFolderName; + protected final Map xmlFilePathLookupByFolderName; + protected final Map xsdFilePathLookupByFolderName; + protected byte[] tableIndexExpectedMD5Sum; + protected byte[] archiveIndexExpectedMD5Sum; + + private final Pattern patternTableFolder = Pattern + .compile("(AVID\\.[A-ZÆØÅ]{2,4}\\.[1-9][0-9]*\\.[1-9][0-9]*)\\\\Tables\\\\(table[0-9]*)"); + private final Pattern patternIndicesFolder = Pattern + .compile("AVID\\.[A-ZÆØÅ]{2,4}\\.[1-9][0-9]*\\.[1-9][0-9]*\\\\Indices"); + + protected SIARDDKFileIndexFile currentFile; + protected StringBuilder currentElementCharacters; + + public SIARDDKFileIndexHandler(Map archiveFolderLookupByFolderName, + Map xsdFilePathLookupByFolderName, + Map xmlFilePathLookupByFolderName) { + this.archiveFolderLookupByFolderName = archiveFolderLookupByFolderName; + this.xsdFilePathLookupByFolderName = xsdFilePathLookupByFolderName; + this.xmlFilePathLookupByFolderName = xmlFilePathLookupByFolderName; + } + + @Override + public void startDocument() throws SAXException { + // no op + } + + @Override + public void endDocument() throws SAXException { + // no op + } + + @Override + public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { + currentElementCharacters = new StringBuilder(); + + if (localName.equals(getFileLocalName())) { + startElementFile(uri, localName, qName, attributes); + } + } + + @Override + public void endElement(String uri, String localName, String qName) throws SAXException { + if (localName.equals(getFileLocalName())) { + endElementFile(); + } else if (localName.equals(getFileNameLocalName())) { + endElementFileName(); + } else if (localName.equals(getFolderNameLocalName())) { + endElementFolderName(); + } else if (localName.equals(getMD5LocalName())) { + endElementMD5(); + } + } + + @Override + public void characters(char[] ch, int start, int length) throws SAXException { + currentElementCharacters.append(ch, start, length); + } + + protected void startElementFile(String uri, String localName, String qName, Attributes attributes) { + this.currentFile = new SIARDDKFileIndexFile(); + } + + private void endElementFile() throws SAXException { + Matcher matcherTableFolder = patternTableFolder.matcher(currentFile.getFolderName()); + if (matcherTableFolder.matches()) { + String folderName = matcherTableFolder.group(2); + String archivePath = FileSystems.getDefault().getPath(matcherTableFolder.group(1)).toString(); + archiveFolderLookupByFolderName.put(folderName, archivePath); + if (currentFile.getFileName().toLowerCase().endsWith(SIARDDKConstants.XML_EXTENSION)) { + if (xmlFilePathLookupByFolderName.containsKey(folderName)) { + throw new SAXException("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX + + " for table files. Multiple entries for the xml file for folder [" + folderName + "]."); + } + xmlFilePathLookupByFolderName.put(folderName, currentFile); + } else { + if (currentFile.getFileName().toLowerCase().endsWith(SIARDDKConstants.XSD_EXTENSION)) { + if (xsdFilePathLookupByFolderName.containsKey(folderName)) { + throw new SAXException("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX + + " for table files. Multiple entries for the xsd file for folder [" + folderName + "]."); + } + xsdFilePathLookupByFolderName.put(folderName, currentFile); + } + } + } else { + Matcher mIndicesFldr = patternIndicesFolder.matcher(currentFile.getFolderName()); + if (mIndicesFldr.matches()) { + // please notice, that this is a rudimentary implementation, only + // considering the files relevant for the SIARDDK import module. + if (currentFile.getFileName().equals(SIARDDKConstants.TABLE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { + tableIndexExpectedMD5Sum = currentFile.getMd5(); + } else if (currentFile.getFileName() + .equals(SIARDDKConstants.ARCHIVE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { + archiveIndexExpectedMD5Sum = currentFile.getMd5(); + } + + } + } + } + + protected void endElementFileName() throws SAXException { + this.currentFile.setFileName(this.currentElementCharacters.toString()); + } + + protected void endElementFolderName() throws SAXException { + this.currentFile.setFolderName(this.currentElementCharacters.toString()); + } + + protected void endElementMD5() throws SAXException { + try { + this.currentFile.setMd5(Hex.decodeHex(this.currentElementCharacters.toString())); + } catch (DecoderException e) { + throw new SAXException("Unable to decode MD5 hex string", e); + } + } + + abstract String getFileLocalName(); + + abstract String getFileNameLocalName(); + + abstract String getFolderNameLocalName(); + + abstract String getMD5LocalName(); + + public byte[] getArchiveIndexExpectedMD5Sum() { + return archiveIndexExpectedMD5Sum; + } + + public byte[] getTableIndexExpectedMD5Sum() { + return tableIndexExpectedMD5Sum; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java index dc863335a..2d25efcb8 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/in/path/SIARDDKPathImportStrategy.java @@ -7,35 +7,44 @@ */ package com.databasepreservation.modules.siard.in.path; -import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; -import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; -import com.databasepreservation.modules.siard.in.read.ReadStrategy; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBElement; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Unmarshaller; -import java.util.List; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.xml.sax.SAXException; - -import javax.xml.XMLConstants; -import javax.xml.transform.stream.StreamSource; -import javax.xml.validation.Schema; -import javax.xml.validation.SchemaFactory; import java.io.IOException; import java.io.InputStream; import java.nio.file.FileSystems; import java.nio.file.Path; +import java.nio.file.Paths; import java.security.InvalidParameterException; -import java.util.HashMap; +import java.util.List; import java.util.Map; -import java.util.regex.Matcher; +import java.util.UUID; import java.util.regex.Pattern; +import javax.xml.XMLConstants; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.parsers.SAXParser; +import javax.xml.parsers.SAXParserFactory; +import javax.xml.transform.stream.StreamSource; +import javax.xml.validation.Schema; +import javax.xml.validation.SchemaFactory; +import javax.xml.validation.ValidatorHandler; + +import org.mapdb.DB; +import org.mapdb.DBMaker; +import org.mapdb.Serializer; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xml.sax.InputSource; +import org.xml.sax.SAXException; +import org.xml.sax.XMLReader; +import org.xml.sax.helpers.DefaultHandler; + +import com.databasepreservation.Constants; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; +import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.in.read.ReadStrategy; +import com.databasepreservation.utils.ConfigUtils; + /** * @author Thomas Kristensen * @@ -45,26 +54,26 @@ * to retrieve md5sums.(The impl. of retrieval of md5sums for the meta * data files are only implemented to the extend that it is needed. ) */ -public abstract class SIARDDKPathImportStrategy implements ContentPathImportStrategy, MetadataPathStrategy { +public abstract class SIARDDKPathImportStrategy extends DefaultHandler + implements ContentPathImportStrategy, MetadataPathStrategy { protected final Logger logger = LoggerFactory.getLogger(ContentPathImportStrategy.class); protected final String importAsSchema; protected final SIARDArchiveContainer mainFolder; protected final ReadStrategy readStrategy; protected final MetadataPathStrategy metadataPathStrategy; - protected final Map xmlFilePathLookupByFolderName = new HashMap(); - protected final Map xsdFilePathLookupByFolderName = new HashMap(); - protected final Map folderNameLookupByTableId = new HashMap(); - protected final Map archiveFolderLookupByFolderName = new HashMap(); - - private FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy; - + protected final DB mapDB; + protected final Map xmlFilePathLookupByFolderName; + protected final Map xsdFilePathLookupByFolderName; + protected final Map folderNameLookupByTableId; + protected final Map archiveFolderLookupByFolderName; protected final Pattern folderSperatorPattern = Pattern.compile("[\\\\\\/]"); + private final Class fileIndexTypeClass; // protected byte[] fileIndexExpectedMD5Sum; --For some reason, no md5sum is // required for fileIndex.xml in the standard - protected byte[] tabelIndexExpectedMD5Sum; + protected byte[] tableIndexExpectedMD5Sum; protected byte[] archiveIndexExpectedMD5Sum; protected boolean fileIndexIsParsed; - private final Class fileIndexTypeClass; + private FileIndexXsdInputStreamStrategy fileIndexXsdInputStreamStrategy; public SIARDDKPathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy readStrategy, MetadataPathStrategy metadataPathStrategy, String importAsSchema, @@ -76,102 +85,78 @@ public SIARDDKPathImportStrategy(SIARDArchiveContainer mainFolder, ReadStrategy this.importAsSchema = importAsSchema; this.fileIndexXsdInputStreamStrategy = fileIndexXsdInputStreamStrategy; this.fileIndexTypeClass = fileIndexTypeClass; + + // Lookups maps + this.mapDB = setupMapDB(); + this.xmlFilePathLookupByFolderName = this.mapDB + .hashMap("xmlFilePathLookupByFolderName", Serializer.STRING, Serializer.JAVA).createOrOpen(); + this.xsdFilePathLookupByFolderName = this.mapDB + .hashMap("xsdFilePathLookupByFolderName", Serializer.STRING, Serializer.JAVA).createOrOpen(); + this.folderNameLookupByTableId = this.mapDB + .hashMap("xsdFilePathLookupByFolderName", Serializer.STRING, Serializer.STRING).createOrOpen(); + this.archiveFolderLookupByFolderName = this.mapDB + .hashMap("archiveFolderLookupByFolderName", Serializer.STRING, Serializer.JAVA).createOrOpen(); } public void parseFileIndexMetadata() throws ModuleException { if (!fileIndexIsParsed) { - JAXBContext context; - try { - context = JAXBContext.newInstance(fileIndexTypeClass.getPackage().getName()); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error loading JAXBContext").withCause(e); - } - SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema xsdSchema = null; + Schema xsdSchema; InputStream xsdStream = fileIndexXsdInputStreamStrategy.getInputStream(this); + ValidatorHandler validatorHandler; + SIARDDKFileIndexHandler fileIndexHandler; try { xsdSchema = schemaFactory.newSchema(new StreamSource(xsdStream)); + validatorHandler = xsdSchema.newValidatorHandler(); + fileIndexHandler = createFileIndexHandler(); + validatorHandler.setContentHandler(fileIndexHandler); } catch (SAXException e) { throw new ModuleException() .withMessage( "Error reading metadata XSD file: " + metadataPathStrategy.getXsdFilePath(SIARDDKConstants.FILE_INDEX)) .withCause(e); } - InputStream reader = null; - D xmlFileIndex; - Unmarshaller unmarshaller; + try { - unmarshaller = context.createUnmarshaller(); - unmarshaller.setSchema(xsdSchema); - reader = readStrategy.createInputStream(mainFolder, - metadataPathStrategy.getXmlFilePath(SIARDDKConstants.FILE_INDEX)); - @SuppressWarnings("unchecked") - JAXBElement jaxbElement = (JAXBElement) unmarshaller.unmarshal(reader); - xmlFileIndex = jaxbElement.getValue(); - } catch (JAXBException e) { - throw new ModuleException().withMessage("Error while Unmarshalling JAXB").withCause(e); + SAXParserFactory saxParserFactory = SAXParserFactory.newInstance(); + saxParserFactory.setValidating(false); + saxParserFactory.setNamespaceAware(true); + SAXParser saxParser = saxParserFactory.newSAXParser(); + XMLReader xmlReader = saxParser.getXMLReader(); + xmlReader.setContentHandler(validatorHandler); + xmlReader.parse(new InputSource(readStrategy.createInputStream(mainFolder, + metadataPathStrategy.getXmlFilePath(SIARDDKConstants.FILE_INDEX)))); + } catch (SAXException | ParserConfigurationException | IOException e) { + throw new ModuleException().withMessage("Error while parsing file index").withCause(e); } finally { try { xsdStream.close(); - if (reader != null) { - reader.close(); - } } catch (IOException e) { logger.debug("Could not close xsdStream", e); } } - Pattern patternTableFolder = Pattern - .compile("(AVID\\.[A-ZÆØÅ]{2,4}\\.[0-9]*\\.[0-9]*)\\\\Tables\\\\(table[0-9]*)"); - - Pattern patternIndicesFolder = Pattern.compile("AVID\\.[A-ZÆØÅ]{2,4}\\.[0-9]*\\.1\\\\Indices"); - - for (T fileInfo : getF(xmlFileIndex)) { - Matcher mTblFldr = patternTableFolder.matcher(getFoN(fileInfo)); - if (mTblFldr.matches()) { - String folderName = mTblFldr.group(2); - Path archivePath = FileSystems.getDefault().getPath(mTblFldr.group(1)); - archiveFolderLookupByFolderName.put(folderName, archivePath); - if (getFiN(fileInfo).toLowerCase().endsWith(SIARDDKConstants.XML_EXTENSION)) { - if (xmlFilePathLookupByFolderName.containsKey(folderName)) { - throw new ModuleException().withMessage("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX - + " for table files. Multiple entries for the xml file for folder [" + folderName + "]."); - } - xmlFilePathLookupByFolderName.put(folderName, fileInfo); - } else { - if (getFiN(fileInfo).toLowerCase().endsWith(SIARDDKConstants.XSD_EXTENSION)) { - if (xsdFilePathLookupByFolderName.containsKey(folderName)) { - throw new ModuleException().withMessage("Inconsistent data in the " + SIARDDKConstants.FILE_INDEX - + " for table files. Multiple entries for the xsd file for folder [" + folderName + "]."); - } - xsdFilePathLookupByFolderName.put(folderName, fileInfo); - } - } - } else { - Matcher mIndicesFldr = patternIndicesFolder.matcher(getFoN(fileInfo)); - if (mIndicesFldr.matches()) { - // please notice, that this is a rudimentary implementation, only - // considering the files relevant for the SIARDDK import module. - if (getFiN(fileInfo).equals(SIARDDKConstants.TABLE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { - tabelIndexExpectedMD5Sum = getMd5(fileInfo); - } else if (getFiN(fileInfo).equals(SIARDDKConstants.ARCHIVE_INDEX + "." + SIARDDKConstants.XML_EXTENSION)) { - archiveIndexExpectedMD5Sum = getMd5(fileInfo); - } - /* - * else { if (fileInfo.getFiN().equals(SIARDDKConstants.FILE_INDEX + "." + - * SIARDDKConstants.XML_EXTENSION)) { fileIndexExpectedMD5Sum = - * fileInfo.getMd5(); } - */ - - } - } - } + tableIndexExpectedMD5Sum = fileIndexHandler.getTableIndexExpectedMD5Sum(); + archiveIndexExpectedMD5Sum = fileIndexHandler.getArchiveIndexExpectedMD5Sum(); + fileIndexIsParsed = true; } } + private DB setupMapDB() { + Path fileDBPath; + String fileDirectoryLocation = ConfigUtils.getProperty(Constants.PROPERTY_UNSET, "dbptk.memory.dir"); + if (fileDirectoryLocation.equals(Constants.PROPERTY_UNSET)) { + fileDBPath = Paths.get(ConfigUtils.getMapDBHomeDirectory().normalize().toAbsolutePath().toString(), + UUID.randomUUID().toString()); + } else { + fileDBPath = Paths.get(fileDirectoryLocation, UUID.randomUUID().toString()); + } + return DBMaker.fileDB(fileDBPath.toFile()).fileDeleteAfterClose().fileMmapEnable().fileMmapEnableIfSupported() + .fileMmapPreclearDisable().closeOnJvmShutdown().make(); + } + @Override public String getLobPath(String basePath, String schemaName, String tableId, String columnId, String lobFileName) { throw new UnsupportedOperationException("Invoking getLobPath(...) is not relevant for SIARDDK."); @@ -225,7 +210,7 @@ public String getTableXMLFilePath(String schemaName, String tableId) throws Modu } public byte[] getTableXMLFileMD5(String schemaName, String tableId) throws ModuleException { - return getMd5(getTableXMLFileInfo(schemaName, tableId)); + return getTableXMLFileInfo(schemaName, tableId).getMd5(); } public byte[] getArchiveIndexExpectedMD5Sum() throws ModuleException { @@ -245,25 +230,25 @@ protected void canLookupXSDFilePath(String folderName) throws ModuleException { } } - protected T getTableXMLFileInfo(String schemaName, String tableId) throws ModuleException { + protected SIARDDKFileIndexFile getTableXMLFileInfo(String schemaName, String tableId) throws ModuleException { canLookupTable(schemaName, tableId); String folderName = folderNameLookupByTableId.get(tableId); canLookupXMLFilePath(folderName); return xmlFilePathLookupByFolderName.get(folderName); } - protected T getTableXSDFileInfo(String schemaName, String tableId) throws ModuleException { + protected SIARDDKFileIndexFile getTableXSDFileInfo(String schemaName, String tableId) throws ModuleException { canLookupTable(schemaName, tableId); String folderName = folderNameLookupByTableId.get(tableId); canLookupXSDFilePath(folderName); return xsdFilePathLookupByFolderName.get(folderName); } - protected String buildPathSansArchiveFolderName(T fileInfo) { + protected String buildPathSansArchiveFolderName(SIARDDKFileIndexFile fileInfo) { Path pathFolderSperatorNeutral = FileSystems.getDefault().getPath("", - folderSperatorPattern.split(getFoN(fileInfo))); + folderSperatorPattern.split(fileInfo.getFolderName())); pathFolderSperatorNeutral = pathFolderSperatorNeutral.subpath(1, pathFolderSperatorNeutral.getNameCount()); - Path pathFolderSperatorNeutralWithFile = pathFolderSperatorNeutral.resolve(getFiN(fileInfo)); + Path pathFolderSperatorNeutralWithFile = pathFolderSperatorNeutral.resolve(fileInfo.getFileName()); return pathFolderSperatorNeutralWithFile.toString(); } @@ -273,14 +258,14 @@ public String getTableXSDFilePath(String schemaName, String tableId) throws Modu } public byte[] getTableXSDFileMD5(String schemaName, String tableId) throws ModuleException { - return getMd5(getTableXSDFileInfo(schemaName, tableId)); + return getTableXSDFileInfo(schemaName, tableId).getMd5(); } public Path getArchiveFolderPath(String schemaName, String tableId) throws ModuleException { canLookupTable(schemaName, tableId); String folderName = folderNameLookupByTableId.get(tableId); assert (archiveFolderLookupByFolderName.containsKey(folderName)); - return archiveFolderLookupByFolderName.get(folderName); + return Path.of(archiveFolderLookupByFolderName.get(folderName)); } @Override @@ -307,13 +292,13 @@ public String getXsdResourcePath(String filename) throws InvalidParameterExcepti * fileIndexExpectedMD5Sum; } */ - public byte[] getTabelIndexExpectedMD5Sum() throws ModuleException { - if (tabelIndexExpectedMD5Sum == null && fileIndexIsParsed) { + public byte[] getTableIndexExpectedMD5Sum() throws ModuleException { + if (tableIndexExpectedMD5Sum == null && fileIndexIsParsed) { throw new ModuleException() .withMessage("Parsing of " + SIARDDKConstants.FILE_INDEX + "." + SIARDDKConstants.XML_EXTENSION + " did not provide a md5sum for " + SIARDDKConstants.TABLE_INDEX + "." + SIARDDKConstants.XML_EXTENSION); } - return tabelIndexExpectedMD5Sum; + return tableIndexExpectedMD5Sum; } /** @@ -330,7 +315,7 @@ public SIARDArchiveContainer getMainFolder() { return mainFolder; } - abstract byte[] getMd5(T fileInfo); + abstract SIARDDKFileIndexHandler createFileIndexHandler(); abstract List getF(D fileIndex); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java index 47163f158..a9707405e 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKContentExportStrategy.java @@ -7,16 +7,20 @@ */ package com.databasepreservation.modules.siard.out.content; -import java.io.BufferedInputStream; import java.io.BufferedWriter; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.nio.file.Path; +import java.util.ArrayList; import java.util.List; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.commons.codec.binary.Hex; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.IOUtils; import org.jdom2.Document; import org.jdom2.Element; @@ -36,6 +40,7 @@ import com.databasepreservation.model.structure.ColumnStructure; import com.databasepreservation.model.structure.SchemaStructure; import com.databasepreservation.model.structure.TableStructure; +import com.databasepreservation.modules.siard.SIARDDKModuleFactory; import com.databasepreservation.modules.siard.common.LargeObject; import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDConstants; @@ -45,6 +50,12 @@ import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.path.ContentPathExportStrategy; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import com.databasepreservation.modules.siard.services.conversion.BypassedLobReporter; +import com.databasepreservation.modules.siard.services.conversion.LobConversionAuditor; +import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.DbptkContext; +import com.fasterxml.jackson.databind.ObjectMapper; public class SIARDDKContentExportStrategy implements ContentExportStrategy { @@ -69,6 +80,10 @@ public class SIARDDKContentExportStrategy implements ContentExportStrategy { private final LOBsTracker lobsTracker; private final MimetypeHandler mimetypeHandler; + private final LobConversionAuditor auditor; + private final BypassedLobReporter bypassedLobReporter; + private final ObjectMapper mapper; + private Reporter reporter; public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { @@ -85,6 +100,15 @@ public SIARDDKContentExportStrategy(SIARDDKExportModule siarddkExportModule) { baseContainer = siarddkExportModule.getMainContainer(); writeStrategy = siarddkExportModule.getWriteStrategy(); lobsTracker = siarddkExportModule.getLobsTracker(); + + this.mapper = new ObjectMapper(); + Path exportRoot = baseContainer.getPath().getParent(); + String archiveName = baseContainer.getPath().getFileName().toString(); + this.auditor = new LobConversionAuditor(exportRoot, archiveName); + + String targetLobFormat = siarddkExportModule.getExportModuleArgs() + .getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_TARGET_FORMAT, "image/tiff"); + this.bypassedLobReporter = new BypassedLobReporter(exportRoot, archiveName, targetLobFormat); } @Override @@ -287,16 +311,14 @@ public Row tableRow(Row row) throws ModuleException { binaryCell.cleanResources(); } } else { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } } else { // cell must contain BLOB or CLOB if (cell instanceof NullCell) { - tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); + whiteNilCell(columnIndex); } else if (cell instanceof SimpleCell) { // CLOB is not NULL @@ -317,76 +339,169 @@ public Row tableRow(Row row) throws ModuleException { } else if (cell instanceof BinaryCell) { // BLOB case - final BinaryCell binaryCell = (BinaryCell) cell; + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; - // BLOB is not NULL + // ------------------------------------------------------------- + // BLOB EXTRACTION DELEGATION + // ------------------------------------------------------------- + if (mimeType.equals("application/zip")) { + processConvertedLobArchive(binaryCell, row.getIndex(), columnIndex); + } else { + processRawLobFile(binaryCell, columnIndex); + } + } else { + // never happens + } + } + } - double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); - lobsTracker.addLOB(lobSizeMB); // Only if LOB not NULL + tableXmlWriter.append(TAB).append("\n"); - // Determine the mimetype (Tika should use an inputstream which - // supports marks) + } catch (IOException e) { + throw new ModuleException().withMessage("Could not write row " + row.toString()).withCause(e); + } - InputStream is = new BufferedInputStream(binaryCell.createInputStream()); - // Removed because TIKA was a security vulnerability and this feature was not - // needed/not fully implemented (see #341) - String mimeType = "unsupported"; - IOUtils.closeQuietly(is); + return row; + } - // Archive BLOB - simultaneous writing always supported for - // SIARDDK + private void processRawLobFile(BinaryCell binaryCell, int columnIndex) throws ModuleException, IOException { + String mimeType = binaryCell.getMimeType() != null ? binaryCell.getMimeType() : "unsupported"; + String fileExtension; + if (mimetypeHandler.isMimetypeAllowed(mimeType)) { + fileExtension = mimetypeHandler.getFileExtension(mimeType); + } else { + logger.warn( + "Found BLOB with unsupported mimetype '{}' in table {}, column {}. archiving as .bin file.", + mimeType, tableCounter, columnIndex); + fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; + foundUnknownMimetype = true; + } - tableXmlWriter.append(TAB).append(TAB).append("") - .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + double lobSizeMB = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeMB); - String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1); - String fileExtension; - if (mimetypeHandler.isMimetypeAllowed(mimeType)) { - fileExtension = mimetypeHandler.getFileExtension(mimeType); - } else { - fileExtension = SIARDDKConstants.UNKNOWN_MIMETYPE_BLOB_EXTENSION; - // Log (table level) that unknown BLOB mimetype was detected - foundUnknownMimetype = true; - } - path += fileExtension; + String path = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + "1." + fileExtension; + LargeObject blob = new LargeObject(binaryCell, path); - LargeObject blob = new LargeObject(binaryCell, path); + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); + InputStream in = blob.getInputStreamProvider().createInputStream(); + IOUtils.copy(in, out); + IOUtils.closeQuietly(in); + IOUtils.closeQuietly(out); + blob.getInputStreamProvider().cleanResources(); - // Create new FileIndexFileStrategy + writeLobReferenceToXml(columnIndex); - // Write the BLOB - OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, blob.getOutputPath(), writeStrategy); - InputStream in = blob.getInputStreamProvider().createInputStream(); - IOUtils.copy(in, out); - IOUtils.closeQuietly(in); - IOUtils.closeQuietly(out); - blob.getInputStreamProvider().cleanResources(); + String originalFilename = binaryCell.getFile() != null ? FilenameUtils.getName(binaryCell.getFile()).stripTrailing() + : "originalFilename"; + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + originalFilename, fileExtension, null); - // Add file to docIndex (a lot easier to do here even though we - // are dealing with metadata) + SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + } - // TO-DO: obtain (how?) hardcoded values - SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), - "originalFilename", fileExtension, null); + private void processConvertedLobArchive(BinaryCell binaryCell, long rowIndex, int columnIndex) + throws ModuleException { + try { + ConversionReport report = extractReportFromZip(binaryCell); + if (report == null) { + throw new ModuleException().withMessage("Missing conversion_report.json in cell archive."); + } - // Add file to fileIndex - SIARDDKFileIndexFileStrategy.addFile(blob.getOutputPath()); + String fileFromCell = binaryCell.getFile(); + if (fileFromCell != null) { + String filename = FilenameUtils.getName(fileFromCell).stripTrailing(); + report = report.withOriginalFilename(filename); + } - } else { - // never happens + List siardPhysicalPaths = new ArrayList<>(); + int fileCount = 0; + String processedFilesExtension = "tif"; + + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) + continue; + + ArtifactReport artifactMeta = findArtifactMetadata(report.artifacts(), zipEntry.getName()); + + // Check if this file is bypassed + if (artifactMeta == null || artifactMeta.isBypassed()) { + logger.warn("Ignoring bypassed or unknown file: {}. Reason: {}", zipEntry.getName(), + artifactMeta != null ? artifactMeta.errorMessage() : "Not in report"); + continue; } + + // LOB isn't bypassed; add it to tracker now + if (fileCount == 0) { + double lobSizeTotal = ((double) binaryCell.getSize()) / (1024 * 1024); + lobsTracker.addLOB(lobSizeTotal); + } + + String fileExt = mimetypeHandler.getFileExtension(artifactMeta.finalMimeType()); + processedFilesExtension = fileExt; + fileCount++; + + String outputPath = writeLobToSiardStorage(zis, fileCount, fileExt); + siardPhysicalPaths.add(outputPath); } } - tableXmlWriter.append(TAB).append("\n"); + if (fileCount > 0) { + writeLobReferenceToXml(columnIndex); + SIARDDKDocIndexFileStrategy.addDoc(lobsTracker.getLOBsCount(), 0, 1, lobsTracker.getDocCollectionCount(), + report.originalFilename(), processedFilesExtension, null); + } else { + whiteNilCell(columnIndex); + } - } catch (IOException e) { - throw new ModuleException().withMessage("Could not write row " + row.toString()).withCause(e); + ConversionReport enrichedReport = report + .withContext(new DbptkContext(tableCounter, rowIndex, columnIndex, siardPhysicalPaths)); + auditor.appendAuditRecord(enrichedReport); + bypassedLobReporter.appendBypassedRecord(enrichedReport); + + } catch (Exception e) { + throw new ModuleException().withMessage("Failed to process converted ZIP archive").withCause(e); } + } - return row; + private ConversionReport extractReportFromZip(BinaryCell binaryCell) throws Exception { + try (ZipInputStream zis = new ZipInputStream(binaryCell.createInputStream())) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + if (zipEntry.getName().toLowerCase().contains("report")) { + return mapper.readValue(zis.readAllBytes(), ConversionReport.class); + } + } + } + return null; + } + + private ArtifactReport findArtifactMetadata(List artifacts, String fileName) { + if (artifacts == null) + return null; + return artifacts.stream().filter(a -> fileName.equals(a.logicalName())).findFirst().orElse(null); + } + + private String writeLobToSiardStorage(InputStream zis, int fileCount, String extension) throws Exception { + String outputPath = contentPathExportStrategy.getBlobFilePath(-1, -1, -1, -1) + fileCount + "." + extension; + OutputStream out = SIARDDKFileIndexFileStrategy.getLOBWriter(baseContainer, outputPath, writeStrategy); + zis.transferTo(out); + SIARDDKFileIndexFileStrategy.addFile(outputPath); + return outputPath; + } + + private void writeLobReferenceToXml(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("") + .append(Integer.toString(lobsTracker.getLOBsCount())).append("\n"); + } + + private void whiteNilCell(int columnIndex) throws IOException { + tableXmlWriter.append(TAB).append(TAB).append("").append("\n"); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java index 7f960b37c..09d732206 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/content/SIARDDKMimetypeHandler.java @@ -27,18 +27,27 @@ public SIARDDKMimetypeHandler() { mimetypeMap = new HashMap(); mimetypeMap.put("image/tiff", "tif"); mimetypeMap.put("image/jp2", "jp2"); + mimetypeMap.put("image/jpeg2000", "jp2"); + mimetypeMap.put("image/x-jp2", "jp2"); mimetypeMap.put("audio/mpeg", "mp3"); + mimetypeMap.put("audio/mp3", "mp3"); + mimetypeMap.put("audio/x-mpeg", "mp3"); + mimetypeMap.put("audio/mpeg3", "mp3"); mimetypeMap.put("video/mp4", "mpg"); mimetypeMap.put("video/mp2t", "mpg"); - // TO-DO: check mimetypes for MPEG with sa.dk - - // Wave files are missing in fileIndex.xsd - this is an error. Will be - // corrected by sa.dk later - - // mimetypeMap.put("audio/wav", "wav"); - // mimetypeMap.put("audio/x-wav", "wav"); - - // TO-DO: build (how?) GML files + mimetypeMap.put("video/mpeg2", "mpg"); + mimetypeMap.put("video/x-mpeg2", "mpg"); + mimetypeMap.put("video/mpeg", "mpg"); + mimetypeMap.put("video/mpeg4-generic", "mpg"); + mimetypeMap.put("video/x-m4v", "mpg"); + mimetypeMap.put("application/mp4", "mpg"); + mimetypeMap.put("application/gml+xml", "gml"); + mimetypeMap.put("text/gml", "gml"); + mimetypeMap.put("application/xml", "gml"); + mimetypeMap.put("audio/wav", "wav"); + mimetypeMap.put("audio/x-wav", "wav"); + mimetypeMap.put("audio/wave", "wav"); + mimetypeMap.put("audio/vnd.wave", "wav"); } /* diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java index ae10da9ae..04720fbf4 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007DocIndexFileStrategy.java @@ -12,6 +12,8 @@ import dk.sa.xmlns.diark._1_0.docindex.DocIndexType; import dk.sa.xmlns.diark._1_0.docindex.DocumentType; +import dk.sa.xmlns.diark._1_0.docindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +25,11 @@ public SIARDDK1007DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java index a80f8276f..c6e66d8b0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK1007FileIndexFileStrategy.java @@ -18,6 +18,8 @@ import java.util.List; +import dk.sa.xmlns.diark._1_0.fileindex.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,6 +38,11 @@ public SIARDDK1007FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java index c57ae786c..259518690 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128DocIndexFileStrategy.java @@ -7,11 +7,14 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.math.BigInteger; +import java.util.List; + import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocumentType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; -import java.math.BigInteger; -import java.util.List; +import jakarta.xml.bind.JAXBElement; /** * @author António Lindo @@ -23,6 +26,11 @@ public SIARDDK128DocIndexFileStrategy() { super(); } + @Override + JAXBElement createDocIndexTypeRootInstance() { + return new ObjectFactory().createDocIndex(createDocIndexTypeInstance()); + } + @Override DocIndexType createDocIndexTypeInstance() { return new DocIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java index d17924986..723712579 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128FileIndexFileStrategy.java @@ -17,6 +17,8 @@ package com.databasepreservation.modules.siard.out.metadata; import com.databasepreservation.modules.siard.bindings.siard_dk_128.FileIndexType; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.ObjectFactory; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -33,6 +35,11 @@ public SIARDDK128FileIndexFileStrategy() { super(); } + @Override + JAXBElement createFileIndexTypeRootInstance() { + return new ObjectFactory().createFileIndex(createFileIndexTypeInstance()); + } + @Override FileIndexType createFileIndexTypeInstance() { return new FileIndexType(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java new file mode 100644 index 000000000..5b28bd2d3 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDK128MetadataExportStrategy.java @@ -0,0 +1,125 @@ +package com.databasepreservation.modules.siard.out.metadata; + +import java.io.IOException; +import java.io.OutputStream; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.model.structure.DatabaseStructure; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.DocIndexType; +import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; +import com.databasepreservation.modules.siard.common.adapters.SIARDDKAdapter; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; +import com.databasepreservation.modules.siard.out.write.WriteStrategy; + +/** + * + * @author Alexandre Flores + */ +public class SIARDDK128MetadataExportStrategy extends SIARDDKMetadataExportStrategy { + + public SIARDDK128MetadataExportStrategy(SIARDDKExportModule siarddkExportModule, SIARDDKAdapter siarddkAdapter) { + super(siarddkExportModule, siarddkAdapter); + } + + @Override + public void writeMetadataXML(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + // TO-DO: Refactor this into one method in class that can be used by + // SIARDDKDatabaseExportModule also + + // Generate tableIndex.xml + + try { + IndexFileStrategy tableIndexFileStrategy = new SIARDDKTableIndexFileStrategy(lobsTracker, siarddkAdapter); + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.TABLE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal("com.databasepreservation.modules.siard.bindings.siard_dk_128", + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.TABLE_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/tableIndex.xsd", writer, + tableIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing tableIndex.xml to the archive.").withCause(e); + } + + // Generate archiveIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.ARCHIVE_INDEX) != null) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.ARCHIVE_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy archiveIndexFileStrategy = new CommandLineIndexFileStrategy(SIARDDKConstants.ARCHIVE_INDEX, + exportModuleArgs, writer, metadataPathStrategy); + archiveIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing archiveIndex.xml to the archive").withCause(e); + } + } + + // Generate contextDocumentationIndex.xml + + if (exportModuleArgs.get(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX) != null) { + try { + + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + IndexFileStrategy contextDocumentationIndexFileStrategy = new CommandLineIndexFileStrategy( + SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, exportModuleArgs, writer, metadataPathStrategy); + contextDocumentationIndexFileStrategy.generateXML(null); + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing contextDocumentationIndex.xml to the archive") + .withCause(e); + } + } + + if (lobsTracker.getLOBsCount() > 0) { + try { + String path = metadataPathStrategy.getXmlFilePath(SIARDDKConstants.DOC_INDEX); + OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(outputContainer, path, writeStrategy); + + siardMarshaller.marshal(DocIndexType.class, metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.DOC_INDEX), + "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/docIndex.xsd", writer, + SIARDDKDocIndexFileStrategy.generateXML(dbStructure)); + + writer.close(); + + SIARDDKFileIndexFileStrategy.addFile(path); + + } catch (IOException e) { + throw new ModuleException().withMessage("Error writing docIndex.xml to the archive.").withCause(e); + } + } + + createLocalSharedFolder(outputContainer); + } + + @Override + public void writeMetadataXSD(DatabaseStructure dbStructure, SIARDArchiveContainer outputContainer, + WriteStrategy writeStrategy) throws ModuleException { + + // Write contents to Schemas/standard + writeSchemaFile(outputContainer, SIARDDKConstants.XML_SCHEMA, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.TABLE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.ARCHIVE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.CONTEXT_DOCUMENTATION_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.FILE_INDEX, writeStrategy); + writeSchemaFile(outputContainer, SIARDDKConstants.RESEARCH_INDEX, writeStrategy); + if (lobsTracker.getLOBsCount() > 0) { + writeSchemaFile(outputContainer, SIARDDKConstants.DOC_INDEX, writeStrategy); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java index 62e393839..c0df07692 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKDocIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.structure.DatabaseStructure; +import jakarta.xml.bind.JAXBElement; /** * @author Andreas Kring @@ -19,10 +20,10 @@ */ public abstract class SIARDDKDocIndexFileStrategy implements IndexFileStrategy { - private T docIndex; + private JAXBElement docIndex; public SIARDDKDocIndexFileStrategy() { - docIndex = createDocIndexTypeInstance(); + docIndex = createDocIndexTypeRootInstance(); } /* @@ -70,11 +71,13 @@ public D addDoc(int dID, int pID, int mID, int docCollectionNumber, String oFn, setGmlXsd(doc, gmlXsd); } - getDoc(docIndex).add(doc); + getDoc(docIndex.getValue()).add(doc); return doc; } + abstract JAXBElement createDocIndexTypeRootInstance(); + abstract T createDocIndexTypeInstance(); abstract D createDocumentTypeInstance(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java index 17b8a3ed8..beaf91773 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKFileIndexFileStrategy.java @@ -12,6 +12,7 @@ import com.databasepreservation.modules.siard.common.SIARDArchiveContainer; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.write.WriteStrategy; +import jakarta.xml.bind.JAXBElement; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +57,8 @@ public Object generateXML(DatabaseStructure dbStructure) throws ModuleException String foNbase = baseContainer.getName(count - 1).toString(); // e.g. // AVID.SA.19000.1 - T fileIndexType = createFileIndexTypeInstance(); - List fList = getF(fileIndexType); + JAXBElement fileIndexType = createFileIndexTypeRootInstance(); + List fList = getF(fileIndexType.getValue()); for (Map.Entry entry : md5sums.entrySet()) { @@ -168,6 +169,8 @@ public byte[] addFile(String path) { return digest; } + abstract JAXBElement createFileIndexTypeRootInstance(); + abstract T createFileIndexTypeInstance(); abstract D createFileIndexTypeFInstance(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java index 6856cc368..53de790e0 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKMetadataExportStrategy.java @@ -7,6 +7,18 @@ */ package com.databasepreservation.modules.siard.out.metadata; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Map; + +import org.apache.commons.io.IOUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import com.databasepreservation.model.exception.ModuleException; import com.databasepreservation.model.reporters.Reporter; import com.databasepreservation.model.structure.DatabaseStructure; @@ -17,17 +29,6 @@ import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; import com.databasepreservation.modules.siard.out.write.WriteStrategy; -import org.apache.commons.io.IOUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Map; /** * @author Andreas Kring @@ -36,13 +37,13 @@ public class SIARDDKMetadataExportStrategy implements MetadataExportStrategy { private static final Logger LOGGER = LoggerFactory.getLogger(SIARDDKMetadataExportStrategy.class); - private SIARDMarshaller siardMarshaller; - private MetadataPathStrategy metadataPathStrategy; - private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; - private SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; - private Map exportModuleArgs; - private LOBsTracker lobsTracker; - private SIARDDKAdapter siarddkAdapter; + protected SIARDMarshaller siardMarshaller; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; + protected SIARDDKDocIndexFileStrategy SIARDDKDocIndexFileStrategy; + protected Map exportModuleArgs; + protected LOBsTracker lobsTracker; + protected SIARDDKAdapter siarddkAdapter; private Reporter reporter; @@ -164,7 +165,7 @@ public void setOnceReporter(Reporter reporter) { this.reporter = reporter; } - private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) + protected void writeSchemaFile(SIARDArchiveContainer container, String indexFile, WriteStrategy writeStrategy) throws ModuleException { InputStream inputStream = this.getClass().getResourceAsStream(metadataPathStrategy.getXsdResourcePath(indexFile)); @@ -197,7 +198,7 @@ private void writeSchemaFile(SIARDArchiveContainer container, String indexFile, } } - private void createLocalSharedFolder(SIARDArchiveContainer container) { + protected void createLocalSharedFolder(SIARDArchiveContainer container) { Path containerPath = container.getPath(); Path localShared = Paths.get("Schemas/localShared"); File folder = containerPath.resolve(localShared).toFile(); diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java index 07217ab04..fa1e18567 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDDKTableIndexFileStrategy.java @@ -58,9 +58,9 @@ public Object generateXML(DatabaseStructure dbStructure) throws ModuleException // Set dbName - mandatory if (dbStructure.getDbOriginalName() != null) { - siarddkBinding.setDbName(dbStructure.getDbOriginalName()); + siarddkBinding.setDbName(escapeString(dbStructure.getDbOriginalName())); } else { - siarddkBinding.setDbName(dbStructure.getName()); + siarddkBinding.setDbName(escapeString(dbStructure.getName())); } // Set databaseProduct diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java index ec54b1d78..d6efe5353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/SIARDMarshaller.java @@ -33,4 +33,21 @@ public interface SIARDMarshaller { */ public void marshal(String context, String localeSchemaLocation, String JAXBSchemaLocation, OutputStream writer, Object jaxbElement) throws ModuleException; + + /** + * Generate JAXB Marshaller for writing XML object to the archive. + * + * @param archiveClass + * Siard archive class to give JAXB context + * @param localeSchemaLocation + * The locale location of the XML schema for the metadata file. + * @param JAXBSchemaLocation + * The Marshaller.JAXB_SCHEMA_LOCATION. + * @param writer + * The OutputStream to write to. + * @param jaxbElement + * The JAXB element to marshal. + */ + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException; } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java index fa1d86f47..1a8dfc108 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/metadata/StandardSIARDMarshaller.java @@ -12,9 +12,6 @@ import java.io.OutputStream; import javax.xml.XMLConstants; -import jakarta.xml.bind.JAXBContext; -import jakarta.xml.bind.JAXBException; -import jakarta.xml.bind.Marshaller; import javax.xml.transform.stream.StreamSource; import javax.xml.validation.Schema; import javax.xml.validation.SchemaFactory; @@ -26,6 +23,10 @@ import com.databasepreservation.model.exception.ModuleException; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; + public class StandardSIARDMarshaller implements SIARDMarshaller { private static final String ENCODING = "UTF-8"; @@ -78,4 +79,51 @@ public void marshal(String contextStr, String localeSchemaLocation, String JAXBS throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); } } + + @Override + public void marshal(Class archiveClass, String localeSchemaLocation, String JAXBSchemaLocation, + OutputStream writer, Object jaxbElement) throws ModuleException { + + // Set up JAXB marshaller + + JAXBContext context; + try { + context = JAXBContext.newInstance(archiveClass.getPackage().getName(), archiveClass.getClassLoader()); + } catch (JAXBException e) { + throw new ModuleException().withMessage("Error loading JAXBContent").withCause(e); + } + + SchemaFactory schemaFactory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); + Schema xsdSchema = null; + try { + InputStream in = this.getClass().getResourceAsStream(localeSchemaLocation); + xsdSchema = schemaFactory.newSchema(new StreamSource(in)); + in.close(); + } catch (SAXException e) { + throw new ModuleException() + .withMessage("XSD file has errors: " + getClass().getResource(localeSchemaLocation).getPath()).withCause(e); + } catch (IOException e) { + throw new ModuleException().withMessage("Could not close InputStream").withCause(e); + } + + Marshaller m; + + try { + + m = context.createMarshaller(); + m.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, Boolean.TRUE); + m.setProperty(Marshaller.JAXB_ENCODING, ENCODING); + m.setProperty(Marshaller.JAXB_SCHEMA_LOCATION, JAXBSchemaLocation); + + m.setSchema(xsdSchema); + + m.marshal(jaxbElement, writer); + + } catch (JAXBException e) { + if (e.getCause() instanceof SAXParseException) { + LOGGER.error(e.getCause().getMessage()); + } + throw new ModuleException().withMessage("Error while Marshalling JAXB").withCause(e); + } + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java index d251c3570..cac3d926d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK1007DatabaseExportModule.java @@ -9,6 +9,8 @@ import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import dk.sa.xmlns.diark._1_0.fileindex.FileIndexType; + /** * @author António Lindo * @@ -23,4 +25,9 @@ public SIARDDK1007DatabaseExportModule(SIARDDKExportModule siarddkExportModule) String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX; } + + @Override + Class getJAXBContextClass() { + return FileIndexType.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java index 334aa2017..ab8b88353 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128DatabaseExportModule.java @@ -7,6 +7,7 @@ */ package com.databasepreservation.modules.siard.out.output; +import com.databasepreservation.modules.siard.bindings.siard_dk_128.SiardDiark; import com.databasepreservation.modules.siard.constants.SIARDDKConstants; /** @@ -23,4 +24,9 @@ public SIARDDK128DatabaseExportModule(SIARDDKExportModule siarddkExportModule) { String getJAXBContext() { return SIARDDKConstants.JAXB_CONTEXT_FILEINDEX_128; } + + @Override + Class getJAXBContextClass() { + return SiardDiark.class; + } } diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java index 0c70f2856..9d1876645 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDK128ExportModule.java @@ -14,15 +14,17 @@ package com.databasepreservation.modules.siard.out.output; +import java.util.Map; + import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter; import com.databasepreservation.modules.siard.common.path.SIARDDK128MetadataPathStrategy; import com.databasepreservation.modules.siard.common.path.SIARDDKMetadataPathStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128DocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDK128FileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDDK128MetadataExportStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKDocIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; import com.databasepreservation.modules.siard.out.metadata.SIARDDKMetadataExportStrategy; -import java.util.Map; /** * @author António Lindo @@ -51,7 +53,7 @@ SIARDDKMetadataPathStrategy createSIARDDKMetadataPathStrategyInstance() { @Override SIARDDKMetadataExportStrategy createSIARDDKMetadataExportStrategyInstance() { - return new SIARDDKMetadataExportStrategy(this, new SIARDDK128Adapter()); + return new SIARDDK128MetadataExportStrategy(this, new SIARDDK128Adapter()); } @Override diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java index e9a85f2cc..d750ca8c2 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKDatabaseExportModule.java @@ -7,33 +7,78 @@ */ package com.databasepreservation.modules.siard.out.output; -import com.databasepreservation.model.exception.ModuleException; -import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; -import com.databasepreservation.modules.siard.constants.SIARDDKConstants; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; -import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; -import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; -import org.apache.commons.io.FileUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +import org.apache.commons.io.FileUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.common.io.providers.PathInputStreamProvider; +import com.databasepreservation.model.data.BinaryCell; +import com.databasepreservation.model.data.Cell; +import com.databasepreservation.model.data.Row; +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.SIARDDKModuleFactory; +import com.databasepreservation.modules.siard.common.path.MetadataPathStrategy; +import com.databasepreservation.modules.siard.constants.SIARDDKConstants; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKContextDocumentationWriter; +import com.databasepreservation.modules.siard.out.metadata.SIARDDKFileIndexFileStrategy; +import com.databasepreservation.modules.siard.out.metadata.SIARDMarshaller; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionService; +import com.databasepreservation.modules.siard.services.conversion.HttpLobConversionServiceException; +import com.databasepreservation.modules.siard.services.conversion.LobConversionService; +import com.databasepreservation.modules.siard.services.conversion.TempFileTracker; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.utils.ConfigUtils; /** + * Handles database export pipeline matching SIARD-DK compliance, incorporating + * high-throughput asynchronous LOB streaming conversion. + * * @author Andreas Kring * */ public abstract class SIARDDKDatabaseExportModule extends SIARDExportDefault { - private SIARDDKExportModule siarddkExportModule; + private final SIARDDKExportModule siarddkExportModule; private static final Logger logger = LoggerFactory.getLogger(SIARDDKDatabaseExportModule.class); + private ExecutorService executorService; + private TempFileTracker tempFileTracker; + private LobConversionService conversionService; + private String targetLobFormat; + private static final Integer MAX_QUEUE_SIZE = ConfigUtils.getProperty(100, "dbptk.siarddk.export.maxQueueSize"); + + // Resilient Pipeline architecture attributes + private BlockingQueue> pendingRowsQueue; + private ExecutorService writerExecutor; + private Future writerTask; + private final AtomicReference writerError = new AtomicReference<>(); + + /** + * Data context tuple to link rows with their transient extracted disk paths. + */ + private record ProcessedRowContext(Row row, List transientPaths) { + } + public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { super(siarddkExportModule.getContentExportStrategy(), siarddkExportModule.getMainContainer(), siarddkExportModule.getWriteStrategy(), siarddkExportModule.getMetadataExportStrategy(), null); @@ -45,6 +90,29 @@ public SIARDDKDatabaseExportModule(SIARDDKExportModule siarddkExportModule) { public void initDatabase() throws ModuleException { super.initDatabase(); + Map exportModuleArgs = siarddkExportModule.getExportModuleArgs(); + boolean isLobConversionEnabled = Boolean + .parseBoolean(exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENABLED, "false")); + + if (isLobConversionEnabled) { + this.tempFileTracker = new TempFileTracker(); + String apiEndpoint = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_ENDPOINT, + "http://localhost:8087"); + this.targetLobFormat = exportModuleArgs.getOrDefault(SIARDDKModuleFactory.PARAMETER_LOB_CONVERSION_TARGET_FORMAT, + "image/tiff"); + + this.conversionService = new HttpLobConversionService(apiEndpoint, this.targetLobFormat, this.tempFileTracker); + logger.info("LOB conversion service enabled. Endpoint: '{}', Target Format: '{}'", apiEndpoint, + this.targetLobFormat); + } else { + this.conversionService = null; + this.tempFileTracker = null; + logger.info("LOB conversion service is disabled."); + } + + this.executorService = Executors.newVirtualThreadPerTaskExecutor(); + this.writerExecutor = Executors.newSingleThreadExecutor(); // Dedicated single-thread pipeline consumer + // Get docID info from the command line and add these to the LOBsTracker Path pathToArchive = siarddkExportModule.getMainContainer().getPath(); @@ -82,8 +150,39 @@ public void initDatabase() throws ModuleException { } } + @Override + public void handleDataOpenTable(String tableId) throws ModuleException { + logger.debug("Opening table '{}'. Initializing asynchronous pipeline...", tableId); + this.pendingRowsQueue = new LinkedBlockingQueue<>(MAX_QUEUE_SIZE); + this.writerError.set(null); + startAsyncWriter(); + super.handleDataOpenTable(tableId); + } + + @Override + public void handleDataRow(Row row) throws ModuleException { + enqueueRow(row); + } + + @Override + public void handleDataCloseTable(String tableId) throws ModuleException { + logger.debug("Closing table '{}'. Draining remaining items in the pipeline...", tableId); + stopAsyncWriter(); + super.handleDataCloseTable(tableId); + } + @Override public void finishDatabase() throws ModuleException { + if (executorService != null && !executorService.isShutdown()) { + executorService.shutdown(); + } + if (writerExecutor != null && !writerExecutor.isShutdown()) { + writerExecutor.shutdown(); + } + if (tempFileTracker != null) { + tempFileTracker.cleanupAll(); + } + super.finishDatabase(); // Write ContextDocumentation to archive @@ -117,7 +216,8 @@ public void finishDatabase() throws ModuleException { OutputStream writer = SIARDDKFileIndexFileStrategy.getWriter(siarddkExportModule.getMainContainer(), path, siarddkExportModule.getWriteStrategy()); - siardMarshaller.marshal(getJAXBContext(), metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), + siardMarshaller.marshal(getJAXBContextClass(), + metadataPathStrategy.getXsdResourcePath(SIARDDKConstants.FILE_INDEX), "http://www.sa.dk/xmlns/diark/1.0 ../Schemas/standard/fileIndex.xsd", writer, SIARDDKFileIndexFileStrategy.generateXML(null)); @@ -125,8 +225,187 @@ public void finishDatabase() throws ModuleException { } catch (IOException e) { throw new ModuleException().withMessage("Error writing fileIndex to the archive.").withCause(e); } + } + /** + * Starts the sequential pipeline background consumer thread. + */ + private void startAsyncWriter() { + this.writerTask = writerExecutor.submit(() -> { + try { + while (!Thread.currentThread().isInterrupted()) { + logger.debug("Consumer thread is waiting to take the oldest row from the queue..."); + Future future = pendingRowsQueue.take(); // Enforces strict sequential order + + logger.debug("Oldest row taken. Awaiting its Virtual Thread completion (LOB HTTP boundary)..."); + ProcessedRowContext context = future.get(); // Awaits specific LOB HTTP processing boundary + + if (context == null) { + logger.debug("<< DEQUEUED: Poison Pill received. Safely shutting down the consumer thread."); + break; + } + + super.handleDataRow(context.row()); + + // Alleviate disk pressure by wiping extracted structures instantly after XML + // writing + cleanupTransientPaths(context.transientPaths()); + logger.debug("<< DEQUEUED: Row [{}] removed from queue and successfully written. Current size: {}/{}", + context.row().getIndex(), pendingRowsQueue.size(), MAX_QUEUE_SIZE); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Pipeline consumer thread was interrupted and is shutting down."); + } catch (ExecutionException e) { + logger.error("A background conversion task failed critically: {}", e.getCause().getMessage()); + writerError.set(e.getCause()); + } catch (Exception e) { + logger.error("An unexpected error occurred during sequential writing.", e); + writerError.set(e); + } + }); + } + + /** + * Gracefully drains the remaining queue, safely stops the consumer and monitors + * failures. + */ + private void stopAsyncWriter() throws ModuleException { + if (writerError.get() == null) { + try { + logger.debug("|| TABLE END: Injecting Poison Pill into the queue and waiting for consumer to finish..."); + + Future poisonPill = executorService.submit(() -> null); + + while (!pendingRowsQueue.offer(poisonPill, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) + break; + } + + if (writerTask != null) { + writerTask.get(); + } + + } catch (InterruptedException | ExecutionException e) { + Thread.currentThread().interrupt(); + throw new ModuleException().withMessage("Failed to cleanly stop the background writer").withCause(e); + } + } + + // Purge any remaining futures in case of a catastrophic error + if (pendingRowsQueue != null && !pendingRowsQueue.isEmpty()) { + for (Future future : pendingRowsQueue) { + future.cancel(true); + } + pendingRowsQueue.clear(); + } + + if (writerError.get() != null) { + throw new ModuleException().withMessage("Row writing pipeline aborted").withCause(writerError.get()); + } + } + + /** + * Pushes rows into the pipeline, throwing fast if the writer task fails, and + * using non-deadlocking backpressure. + */ + private void enqueueRow(Row row) throws ModuleException { + if (writerError.get() != null) { + throw new ModuleException().withMessage("Pipeline execution halted due to previous background failure") + .withCause(writerError.get()); + } + + Callable conversionTask = () -> processRowAsync(row); + Future future = executorService.submit(conversionTask); + + try { + // Prevents deadlocks if the consumer thread crashes while queue is maxed out + boolean waitingLogged = false; + while (!pendingRowsQueue.offer(future, 500, TimeUnit.MILLISECONDS)) { + if (writerError.get() != null) { + future.cancel(true); + throw new ModuleException().withMessage("Pipeline halted while enqueuing row").withCause(writerError.get()); + } + + if (!waitingLogged) { + logger.debug("|| PAUSED: Queue is full. Suspended database reading. Waiting for space to enqueue row [{}]...", + row.getIndex()); + waitingLogged = true; + } + } + logger.debug(">> ENQUEUED: Row [{}] entered the queue. Current size: {}/{}", row.getIndex(), + pendingRowsQueue.size(), MAX_QUEUE_SIZE); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + future.cancel(true); + throw new ModuleException().withMessage("Row enqueuing process was interrupted").withCause(e); + } + } + + /** + * Processes row columns concurrently on Virtual Threads mapping extracted files + * for lifecycle control. + */ + private ProcessedRowContext processRowAsync(Row row) throws ModuleException { + List cells = row.getCells(); + List transientPaths = new ArrayList<>(); + + if (this.conversionService != null) { + for (int i = 0; i < cells.size(); i++) { + Cell cell = cells.get(i); + + if (cell instanceof BinaryCell binCell) { + try (InputStream originalStream = binCell.createInputStream()) { + ConversionResult result = conversionService.convertLob(cell.getId(), originalStream); + + // TODO: Handle multiple files per cell if needed. Currently assumes single file + // output. + BinaryCell newCell = new BinaryCell(cell.getId(), new PathInputStreamProvider(result.zipFile()), + "application/zip"); + newCell.setFile(binCell.getFile()); + cells.set(i, newCell); + + // Track extracted parts to clean them individually later + transientPaths.addAll(result.convertedFiles()); + transientPaths.add(result.reportFile()); + transientPaths.add(result.zipFile()); + if (!result.convertedFiles().isEmpty()) { + transientPaths.add(result.convertedFiles().getFirst().getParent()); // directory container + } + } catch (IOException | InterruptedException | HttpLobConversionServiceException e) { + String statusCodeInfo = ""; + if (e instanceof HttpLobConversionServiceException apiEx && apiEx.getHttpStatusCode() != null) { + statusCodeInfo = " (HTTP " + apiEx.getHttpStatusCode() + ")"; + } + + String errorMsg = String.format( + "Conversion failed for cell '%s' in row %d%s. " + "Pipeline continuing with original file. Detail: %s", + cell.getId(), row.getIndex(), statusCodeInfo, e.getMessage()); + + logger.error(errorMsg); + } + } + } + } + row.setCells(cells); + + long readyCount = pendingRowsQueue.stream().filter(Future::isDone).count(); + logger.debug("== READY: Row [{}] finished conversion. Currently {} ready rows waiting in queue.", row.getIndex(), + readyCount + 1); + + return new ProcessedRowContext(row, transientPaths); + } + + private void cleanupTransientPaths(List paths) { + if (paths == null || tempFileTracker == null) { + return; + } + for (Path path : paths) { + tempFileTracker.deleteEarly(path); + } } abstract String getJAXBContext(); -} + + abstract Class getJAXBContextClass(); +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java index 7fd8d54d0..85c7e2dc6 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/output/SIARDDKExportModule.java @@ -35,14 +35,14 @@ * */ public abstract class SIARDDKExportModule { - private MetadataExportStrategy metadataExportStrategy; - private SIARDArchiveContainer mainContainer; - private ContentExportStrategy contentExportStrategy; - private WriteStrategy writeStrategy; - private ContentPathExportStrategy contentPathExportStrategy; - private MetadataPathStrategy metadataPathStrategy; - private SIARDMarshaller siardMarshaller; - private LOBsTracker lobsTracker; + protected MetadataExportStrategy metadataExportStrategy; + protected SIARDArchiveContainer mainContainer; + protected ContentExportStrategy contentExportStrategy; + protected WriteStrategy writeStrategy; + protected ContentPathExportStrategy contentPathExportStrategy; + protected MetadataPathStrategy metadataPathStrategy; + protected SIARDMarshaller siardMarshaller; + protected LOBsTracker lobsTracker; private Map exportModuleArgs; private SIARDDKFileIndexFileStrategy SIARDDKFileIndexFileStrategy; diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java index dcbb3fce0..6b197218d 100644 --- a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/out/path/SIARDDKContentPathExportStrategy.java @@ -7,10 +7,11 @@ */ package com.databasepreservation.modules.siard.out.path; +import org.apache.commons.lang3.NotImplementedException; + import com.databasepreservation.modules.siard.constants.SIARDDKConstants; import com.databasepreservation.modules.siard.out.content.LOBsTracker; import com.databasepreservation.modules.siard.out.output.SIARDDKExportModule; -import org.apache.commons.lang3.NotImplementedException; /** * @author Andreas Kring @@ -24,14 +25,13 @@ public class SIARDDKContentPathExportStrategy implements ContentPathExportStrate private static final String SCHEMA_DIR = "schema"; private static final String DOCUMENT_DIR = "Documents"; private static final String DOC_COLLECTION = "docCollection"; - private static final String fileCount = "1"; // Design decision private LOBsTracker lobsTracker; public SIARDDKContentPathExportStrategy(LOBsTracker loBsTracker) { this.lobsTracker = loBsTracker; } - + public SIARDDKContentPathExportStrategy(SIARDDKExportModule siarddkExportModule) { this.lobsTracker = siarddkExportModule.getLobsTracker(); } @@ -76,8 +76,7 @@ public String getBlobFilePath(int schemaIndex, int tableIndex, int columnIndex, // Note: code assumes one file in each folder return new StringBuilder().append(DOCUMENT_DIR).append(SIARDDKConstants.FILE_SEPARATOR).append(DOC_COLLECTION) .append(docCollectionCount).append(SIARDDKConstants.FILE_SEPARATOR).append(LOBsCount) - .append(SIARDDKConstants.FILE_SEPARATOR).append(fileCount).append(SIARDDKConstants.FILE_EXTENSION_SEPARATOR) - .toString(); + .append(SIARDDKConstants.FILE_SEPARATOR).toString(); } // Not used in SIARDDK diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java new file mode 100644 index 000000000..976d22583 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/BypassedLobReporter.java @@ -0,0 +1,70 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.report.ArtifactReport; +import com.databasepreservation.modules.siard.services.conversion.model.report.BypassedLobEntry; +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * @author Vitor Leite + */ +public class BypassedLobReporter { + private static final Logger logger = LoggerFactory.getLogger(BypassedLobReporter.class); + private static final String REASON_CONVERSION_FAILED = "Conversion failed"; + private static final String REASON_FILTERED_FORMAT = "Retained the original format due to SIARD-DK specification"; + + private final ObjectMapper mapper; + private final Path reportFilePath; + private final String targetLobFormat; + + public BypassedLobReporter(Path baseExportDirectory, String archiveName, String targetLobFormat) { + this.mapper = new ObjectMapper(); + this.targetLobFormat = targetLobFormat; + String fileName = archiveName + "_bypassed_lob_report.jsonl"; + this.reportFilePath = baseExportDirectory.resolve(fileName); + } + + public void appendBypassedRecord(ConversionReport report) { + if (report == null || report.artifacts() == null) + return; + + for (ArtifactReport artifact : report.artifacts()) { + String reason = resolveBypassReason(artifact); + if (reason == null) + continue; + + BypassedLobEntry entry = new BypassedLobEntry(artifact.logicalName(), artifact.originalMimeType(), + artifact.finalMimeType(), artifact.isBypassed(), artifact.errorMessage(), reason, report.dbptkContext()); + append(entry); + } + } + + private String resolveBypassReason(ArtifactReport artifact) { + if (artifact.isBypassed()) { + return REASON_CONVERSION_FAILED; + } + + if (!targetLobFormat.equalsIgnoreCase(artifact.finalMimeType())) { + return REASON_FILTERED_FORMAT; + } + + return null; + } + + private void append(BypassedLobEntry entry) { + try { + String jsonLine = mapper.writeValueAsString(entry); + Files.writeString(reportFilePath, jsonLine + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + logger.error("Failed to append bypassed lob entry to report.", e); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java new file mode 100644 index 000000000..9dd6f3211 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionService.java @@ -0,0 +1,268 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.ByteArrayInputStream; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.SequenceInputStream; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpRequest.BodyPublishers; +import java.net.http.HttpResponse; +import java.net.http.HttpResponse.BodyHandlers; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Random; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; +import com.databasepreservation.modules.siard.services.conversion.model.JobStatusResponse; +import com.databasepreservation.modules.siard.services.conversion.model.JobSubmissionResponse; +import com.databasepreservation.utils.ConfigUtils; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Handles communication with the external format conversion REST API. Uses + * asynchronous polling and in-memory multipart streaming to process large + * objects efficiently. + * + * @author Gabriel Barros + */ +public class HttpLobConversionService implements LobConversionService { + + private static final Logger log = LoggerFactory.getLogger(HttpLobConversionService.class); + + private static final int MAX_NETWORK_RETRIES = ConfigUtils.getProperty(3, + "dbptk.service.lob.conversion.networkRetries"); + private static final int BASE_POLLING_INTERVAL_MS = ConfigUtils.getProperty(2000, + "dbptk.service.lob.conversion.basePollingIntervalMs"); + private static final int MAX_POLLING_ATTEMPTS = ConfigUtils.getProperty(600, + "dbptk.service.lob.conversion.maxPollingAttempts"); + private static final int CONNECTION_TIMEOUT_SECONDS = ConfigUtils.getProperty(60, + "dbptk.service.lob.conversion.connectionTimeoutSeconds"); + + private static final String CRLF = "\r\n"; + + private final HttpClient httpClient; + private final String baseUrl; + private final String targetFormat; + private final ObjectMapper objectMapper; + private final TempFileTracker fileTracker; + private final Random random = new Random(); + + public HttpLobConversionService(String baseUrl, String targetFormat, TempFileTracker fileTracker) { + this.baseUrl = baseUrl; + this.targetFormat = targetFormat; + this.fileTracker = fileTracker; + this.objectMapper = new ObjectMapper().configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + this.httpClient = HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(CONNECTION_TIMEOUT_SECONDS)).build(); + } + + @Override + public ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { + log.debug("Initiating conversion pipeline for cell: {}", cellId); + String jobId = submitJob(cellId, inputStream); + JobStatus status = waitForCompletion(cellId, jobId); + ConversionResult result = downloadResult(cellId, jobId); + if (JobStatus.COMPLETED.equals(status)) { + deleteJob(cellId, jobId); + } + return result; + } + + private String submitJob(String cellId, InputStream inputStream) + throws IOException, InterruptedException, HttpLobConversionServiceException { + String boundary = "DbptkBoundary" + System.currentTimeMillis(); + + String header = buildMultipartHeader(boundary, cellId); + String footer = CRLF + "--" + boundary + "--" + CRLF; + + InputStream headerStream = new ByteArrayInputStream(header.getBytes(StandardCharsets.UTF_8)); + InputStream footerStream = new ByteArrayInputStream(footer.getBytes(StandardCharsets.UTF_8)); + + InputStream multipartStream = new SequenceInputStream(new SequenceInputStream(headerStream, inputStream), + footerStream); + + HttpRequest submitRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs")) + .header("Content-Type", "multipart/form-data; boundary=" + boundary) + .POST(BodyPublishers.ofInputStream(() -> multipartStream)).build(); + + HttpResponse submitResponse = executeWithRetry(submitRequest, BodyHandlers.ofString(), MAX_NETWORK_RETRIES); + + if (submitResponse.statusCode() >= 400) { + log.error("API rejected LOB submission for cell {}. Status: {}, Body: {}", cellId, submitResponse.statusCode(), + submitResponse.body()); + throw new HttpLobConversionServiceException("Failed to submit LOB for cell " + cellId, + submitResponse.statusCode()); + } + + JobSubmissionResponse job = objectMapper.readValue(submitResponse.body(), JobSubmissionResponse.class); + log.debug("Successfully dispatched cell {}. Assigned Job ID: {}", cellId, job.id()); + return job.id(); + } + + private JobStatus waitForCompletion(String cellId, String jobId) + throws IOException, InterruptedException, HttpLobConversionServiceException { + log.debug("Awaiting completion of Job {} (Cell {})", jobId, cellId); + + for (int attempts = 1; attempts <= MAX_POLLING_ATTEMPTS; attempts++) { + HttpRequest statusRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).GET().build(); + HttpResponse statusResponse = executeWithRetry(statusRequest, BodyHandlers.ofString(), + MAX_NETWORK_RETRIES); + + JobStatusResponse response = objectMapper.readValue(statusResponse.body(), JobStatusResponse.class); + + switch (response.status()) { + case JobStatus.COMPLETED -> { + log.debug("Job {} (Cell {}) completed successfully after {} attempts.", jobId, cellId, attempts); + return response.status(); + } + case JobStatus.FAILED, JobStatus.EVICTED -> { + log.error("API reported terminal failure for Job {} (Cell {}) with status: {}", jobId, cellId, + response.status()); + throw new HttpLobConversionServiceException( + "Server failed to convert cell: " + cellId + " (Status: " + response.status() + ")"); + } + case JobStatus.ACCEPTED, JobStatus.PROCESSING -> { + if (attempts % 30 == 0) { + log.warn("Job {} (Cell {}) is taking unusually long. Current status: {}. Attempt: {}/{}", jobId, cellId, + response.status(), attempts, MAX_POLLING_ATTEMPTS); + } + long sleepTime = BASE_POLLING_INTERVAL_MS + random.nextInt(1000); + Thread.sleep(sleepTime); + } + } + } + log.error("Zombie Job detected. API failed to resolve Job {} (Cell {}) within the maximum polling threshold.", + jobId, cellId); + throw new HttpLobConversionServiceException("Timeout after waiting for conversion of cell: " + cellId); + } + + /** + * Downloads the resulting ZIP and lists its contents, returning the compressed + * file. + */ + private ConversionResult downloadResult(String cellId, String jobId) throws IOException, InterruptedException { + Path zipFile = Files.createTempFile("siarddk_conv_" + cellId + "_" + jobId, ".zip"); + + fileTracker.track(zipFile); + + HttpRequest downloadRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId + "/download")) + .GET().build(); + + executeWithRetry(downloadRequest, BodyHandlers.ofFile(zipFile), MAX_NETWORK_RETRIES); + + return listZipContents(cellId, zipFile); + } + + private void deleteJob(String cellId, String jobId) { + HttpRequest deleteRequest = HttpRequest.newBuilder().uri(URI.create(baseUrl + "/jobs/" + jobId)).DELETE().build(); + + httpClient.sendAsync(deleteRequest, BodyHandlers.discarding()).whenComplete((response, e) -> { + if (response.statusCode() >= 400) { + log.warn("API rejected deletion of Job {} (Cell {}). Status: {}", jobId, cellId, response.statusCode()); + } else { + log.warn("Failed to delete Job {} (Cell {}) after successful download: {}", jobId, cellId, e.getMessage()); + } + }); + } + + private ConversionResult listZipContents(String cellId, Path zipFile) throws IOException { + Path extractionDir = Files.createTempDirectory("siarddk_extracted_" + cellId + "_"); + fileTracker.trackDir(extractionDir); + + Path normalizedExtractionDir = extractionDir.normalize(); + + List convertedFiles = new ArrayList<>(); + Path reportFile = null; + + try (ZipInputStream zis = new ZipInputStream(new FileInputStream(zipFile.toFile()))) { + ZipEntry zipEntry; + while ((zipEntry = zis.getNextEntry()) != null) { + Path extractedFilePath = extractionDir.resolve(zipEntry.getName()).normalize(); + + if (!extractedFilePath.startsWith(normalizedExtractionDir)) { + throw new SecurityException("Corrupted ZIP entry (Zip Slip vulnerability detected): " + zipEntry.getName()); + } + + if (!zipEntry.isDirectory()) { + Files.copy(zis, extractedFilePath, java.nio.file.StandardCopyOption.REPLACE_EXISTING); + + if (zipEntry.getName().toLowerCase().contains("report")) { + reportFile = extractedFilePath; + } else { + convertedFiles.add(extractedFilePath); + } + } + } + } + + if (convertedFiles.isEmpty() || reportFile == null) { + throw new RuntimeException("Downloaded ZIP lacks expected format (at least 1 LOB + Report) for cell: " + cellId); + } + + return new ConversionResult(convertedFiles, reportFile, zipFile); + } + + private HttpResponse executeWithRetry(HttpRequest request, HttpResponse.BodyHandler responseBodyHandler, + int maxRetries) throws InterruptedException, IOException { + Exception lastException = null; + + for (int attempt = 1; attempt <= maxRetries; attempt++) { + try { + HttpResponse response = httpClient.send(request, responseBodyHandler); + + if (response.statusCode() >= 500) { + throw new IOException("Temporary server error: " + response.statusCode() + " - " + response.body()); + } + + return response; + } catch (IOException e) { + lastException = e; + log.warn("Attempt {} failed for {}: {}", attempt, request.uri(), e.getMessage()); + + if (attempt == maxRetries) + break; + + try { + Thread.sleep((long) Math.pow(2, attempt) * 1000); + } catch (InterruptedException ie) { + Thread.currentThread().interrupt(); + throw new IOException("Retry interrupted for URI: " + request.uri(), ie); + } + } + } + throw new IOException("Exhausted all network retries for URI: " + request.uri(), lastException); + } + + private String buildMultipartHeader(String boundary, String cellId) { + StringBuilder sb = new StringBuilder(); + + // Target format part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"targetFormat\"").append(CRLF); + sb.append(CRLF); + sb.append(this.targetFormat).append(CRLF); + + // File part header + sb.append("--").append(boundary).append(CRLF); + sb.append("Content-Disposition: form-data; name=\"file\"; filename=\"lob_").append(cellId).append(".bin\"") + .append(CRLF); + sb.append("Content-Type: application/octet-stream").append(CRLF); + sb.append(CRLF); + + return sb.toString(); + } +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java new file mode 100644 index 000000000..9fc7d9960 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/HttpLobConversionServiceException.java @@ -0,0 +1,33 @@ +package com.databasepreservation.modules.siard.services.conversion; + +/** + * @author Gabriel Barros + */ +public class HttpLobConversionServiceException extends Exception { + + private final Integer httpStatusCode; + + public HttpLobConversionServiceException(String message) { + super(message); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, Throwable cause) { + super(message, cause); + this.httpStatusCode = null; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode) { + super(message); + this.httpStatusCode = httpStatusCode; + } + + public HttpLobConversionServiceException(String message, int httpStatusCode, Throwable cause) { + super(message, cause); + this.httpStatusCode = httpStatusCode; + } + + public Integer getHttpStatusCode() { + return httpStatusCode; + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java new file mode 100644 index 000000000..a5d4d30d1 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionAuditor.java @@ -0,0 +1,41 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardOpenOption; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.databasepreservation.modules.siard.services.conversion.model.report.ConversionReport; +import com.fasterxml.jackson.databind.ObjectMapper; + +/** + * Enriches and persists LOB conversion reports. + * + * @author Gabriel Barros + */ +public class LobConversionAuditor { + private static final Logger logger = LoggerFactory.getLogger(LobConversionAuditor.class); + private final ObjectMapper mapper; + private final Path auditFilePath; + + public LobConversionAuditor(Path baseExportDirectory, String archiveName) { + this.mapper = new ObjectMapper(); + String fileName = archiveName + "_lob_conversion_audit.jsonl"; + this.auditFilePath = baseExportDirectory.resolve(fileName); + } + + public void appendAuditRecord(ConversionReport report) { + if (report == null) + return; + + try { + String jsonLine = mapper.writeValueAsString(report); + Files.writeString(auditFilePath, jsonLine + System.lineSeparator(), StandardOpenOption.CREATE, + StandardOpenOption.APPEND); + } catch (Exception e) { + logger.error("Failed to append conversion report to audit log.", e); + } + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java new file mode 100644 index 000000000..0f26c5b32 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/LobConversionService.java @@ -0,0 +1,12 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.IOException; +import java.io.InputStream; + +import com.databasepreservation.model.exception.ModuleException; +import com.databasepreservation.modules.siard.services.conversion.model.ConversionResult; + +public interface LobConversionService { + ConversionResult convertLob(String cellId, InputStream inputStream) + throws IOException, ModuleException, InterruptedException, HttpLobConversionServiceException; +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java new file mode 100644 index 000000000..ae82d21e0 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/TempFileTracker.java @@ -0,0 +1,95 @@ +package com.databasepreservation.modules.siard.services.conversion; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Comparator; +import java.util.Queue; +import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.stream.Stream; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tracks and manages the lifecycle of temporary files and directories created + * during the LOB conversion process. + * + * @author Gabriel Barros + */ +public class TempFileTracker { + private static final Logger LOGGER = LoggerFactory.getLogger(TempFileTracker.class); + private final Queue trackedPaths = new ConcurrentLinkedQueue<>(); + + /** + * Registers a file or directory path to be tracked for final cleanup. * @param + * path The path to track. + */ + public void track(Path path) { + if (path != null) { + trackedPaths.add(path); + } + } + + /** + * Registers a directory path to be tracked for final cleanup. * @param dirPath + * The directory path to track. + */ + public void trackDir(Path dirPath) { + track(dirPath); + } + + /** + * Deletes a tracked path immediately (including non-empty directories) and + * removes it from the tracking queue to free resources early. * @param path The + * path to delete immediately. + */ + public void deleteEarly(Path path) { + if (path == null) { + return; + } + try { + deleteRecursively(path); + trackedPaths.remove(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary path early: " + path, e); + } + } + + /** + * Deletes all remaining tracked files and directories comprehensively, ensuring + * recursive cleanup of nested content. + */ + public void cleanupAll() { + for (Path path : trackedPaths) { + try { + deleteRecursively(path); + } catch (Exception e) { + LOGGER.warn("Unable to delete temporary file/directory during bulk cleanup: " + path, e); + } + } + trackedPaths.clear(); + } + + /** + * Helper method to perform safe recursive deletion of paths and directories. + */ + private void deleteRecursively(Path path) throws IOException { + if (!Files.exists(path)) { + return; + } + if (Files.isDirectory(path)) { + try (Stream walk = Files.walk(path)) { + walk.sorted(Comparator.reverseOrder()).forEach(p -> { + try { + Files.deleteIfExists(p); + } catch (IOException e) { + LOGGER.warn("Failed to delete nested path: " + p, e); + } + }); + } + } else { + Files.deleteIfExists(path); + } + } +} \ No newline at end of file diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java new file mode 100644 index 000000000..5d229bb2d --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/ConversionResult.java @@ -0,0 +1,10 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +import java.nio.file.Path; +import java.util.List; + +/** + * @author Gabriel Barros + */ +public record ConversionResult(List convertedFiles, Path reportFile, Path zipFile) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java new file mode 100644 index 000000000..5e7107946 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public enum JobStatus { + ACCEPTED, PROCESSING, COMPLETED, FAILED, EVICTED; +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java new file mode 100644 index 000000000..4028cd94f --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobStatusResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public record JobStatusResponse(JobStatus status) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java new file mode 100644 index 000000000..933301871 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/JobSubmissionResponse.java @@ -0,0 +1,7 @@ +package com.databasepreservation.modules.siard.services.conversion.model; + +/** + * @author Gabriel Barros + */ +public record JobSubmissionResponse(String id) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java new file mode 100644 index 000000000..80552d0b5 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ArtifactReport.java @@ -0,0 +1,17 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ArtifactReport(@JsonProperty("logicalName") String logicalName, + @JsonProperty("originalMimeType") String originalMimeType, @JsonProperty("finalMimeType") String finalMimeType, + @JsonProperty("isBypassed") boolean isBypassed, @JsonProperty("complianceStatus") ComplianceStatus complianceStatus, + @JsonProperty("formatHistory") List formatHistory, + @JsonProperty("auditTrail") List auditTrail, @JsonProperty("errorMessage") String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java new file mode 100644 index 000000000..eab47c711 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/AuditTrailStep.java @@ -0,0 +1,17 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.Map; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record AuditTrailStep(@JsonProperty("stepId") String stepId, @JsonProperty("pluginId") String pluginId, + @JsonProperty("agentName") String agentName, @JsonProperty("agentVersion") String agentVersion, + @JsonProperty("agentType") String agentType, @JsonProperty("command") String command, + @JsonProperty("parameters") Map parameters, @JsonProperty("durationMs") long durationMs, + @JsonProperty("successful") boolean successful, @JsonProperty("errorMessage") String errorMessage) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java new file mode 100644 index 000000000..3ec0db906 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/BypassedLobEntry.java @@ -0,0 +1,12 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Vitor Leite + */ +public record BypassedLobEntry(@JsonProperty("logicalName") String logicalName, + @JsonProperty("originalMimeType") String originalMimeType, @JsonProperty("finalMimeType") String finalMimeType, + @JsonProperty("isBypassed") boolean isBypassed, @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("reason") String reason, @JsonProperty("dbptkContext") DbptkContext dbptkContext) { +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java new file mode 100644 index 000000000..8d484c56a --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ComplianceStatus.java @@ -0,0 +1,8 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +/** + * @author Gabriel Barros + */ +public enum ComplianceStatus { + PASSED, PARTIAL, FAILED +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java new file mode 100644 index 000000000..32aaec45f --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/ConversionReport.java @@ -0,0 +1,27 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.databasepreservation.modules.siard.services.conversion.model.JobStatus; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +@JsonIgnoreProperties(ignoreUnknown = true) +public record ConversionReport(@JsonProperty("jobId") String jobId, @JsonProperty("status") JobStatus status, + @JsonProperty("originalFilename") String originalFilename, + @JsonProperty("totalArtifactsProduced") Integer totalArtifactsProduced, + @JsonProperty("artifacts") List artifacts, @JsonProperty("errorMessage") String errorMessage, + @JsonProperty("dbptkContext") DbptkContext dbptkContext) { + public ConversionReport withContext(DbptkContext context) { + return new ConversionReport(jobId, status, originalFilename, totalArtifactsProduced, artifacts, errorMessage, + context); + } + + public ConversionReport withOriginalFilename(String newFilename) { + return new ConversionReport(jobId, status, newFilename, totalArtifactsProduced, artifacts, errorMessage, + dbptkContext); + } +} diff --git a/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java new file mode 100644 index 000000000..900387b56 --- /dev/null +++ b/dbptk-modules/dbptk-module-siard/src/main/java/com/databasepreservation/modules/siard/services/conversion/model/report/DbptkContext.java @@ -0,0 +1,12 @@ +package com.databasepreservation.modules.siard.services.conversion.model.report; + +import java.util.List; + +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author Gabriel Barros + */ +public record DbptkContext(@JsonProperty("tableIndex") int tableIndex, @JsonProperty("rowIndex") long rowIndex, + @JsonProperty("columnIndex") int columnIndex, @JsonProperty("siardPaths") List siardPaths) { +} diff --git a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java index 400746220..0efb41e01 100644 --- a/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java +++ b/dbptk-modules/dbptk-module-sql-server/src/main/java/com/databasepreservation/modules/sqlserver/in/SQLServerDatatypeImporter.java @@ -34,6 +34,19 @@ protected Type getBinaryType(String typeName, int columnSize, int decimalDigits, if (typeName.contains("timestamp") || typeName.contains("rowversion")) { return super.getBinaryType(typeName, 8, decimalDigits, numPrecRadix); } + return super.getBinaryType(typeName, columnSize, decimalDigits, numPrecRadix); } + + @Override + protected Type getVarbinaryType(String typeName, int columnSize, int decimalDigits, int numPrecRadix) { + if (typeName.equalsIgnoreCase("varbinary") && (columnSize == 2147483647 || columnSize == -1)) { + Type type = new SimpleTypeBinary(columnSize); + type.setSql99TypeName("BINARY LARGE OBJECT"); + type.setSql2008TypeName("BINARY LARGE OBJECT"); + return type; + } + + return super.getVarbinaryType(typeName, columnSize, decimalDigits, numPrecRadix); + } }