Merge changes from development

dbptk-core/src/test/java/com/databasepreservation/modules/siard/out/metadata/TestSIARDDK1007TableIndexFileStrategy.java

@@ -9,10 +9,10 @@
 
 import static org.testng.AssertJUnit.assertEquals;
 
-import com.databasepreservation.modules.siard.common.adapters.SIARDDK1007Adapter;
-import com.databasepreservation.modules.siard.common.adapters.SIARDDK128Adapter;
 import org.testng.annotations.Test;
 
+import com.databasepreservation.modules.siard.common.adapters.SIARDDK1007Adapter;
+
 /**
  * @author Andreas Kring <andreas@magenta.dk>
  *

dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardDKTestWrapper.java

@@ -11,6 +11,9 @@
 import java.util.ArrayList;
 import java.util.Iterator;
 
+import com.databasepreservation.Main;
+import com.databasepreservation.utils.ConfigUtils;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
@@ -30,6 +33,11 @@
 @Test(groups = {"siarddk-roundtrip"})
 public class SiardDKTestWrapper {
 
+  @BeforeClass
+  public void init() {
+    ConfigUtils.initialize();
+  }
+
   @DataProvider
   public Iterator<Object[]> siardVersionsProvider() {
     ArrayList<Object[]> tests = new ArrayList<Object[]>();

dbptk-core/src/test/java/com/databasepreservation/testing/integration/siard/SiardTest.java

@@ -36,6 +36,7 @@
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
+import com.databasepreservation.common.io.providers.TemporaryPathInputStreamProvider;
 import com.databasepreservation.model.data.BinaryCell;
 import com.databasepreservation.model.data.Cell;
 import com.databasepreservation.model.data.Row;
@@ -539,15 +540,15 @@ protected DatabaseStructure generateDatabaseStructure() throws ModuleException,
         new Row(1,
           Arrays.asList(new SimpleCell("table02.col121.0", "1"), new SimpleCell("table02.col122.0", "3"),
             new SimpleCell("table02.col123.0", "abc"), new SimpleCell("table02.col124.0", "def"),
-            new BinaryCell("table02.col125.0", newBlob()))),
+            new BinaryCell("table02.col125.0", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))),
         new Row(2,
           Arrays.asList(new SimpleCell("table02.col121.1", "2"), new SimpleCell("table02.col122.1", "1"),
             new SimpleCell("table02.col123.1", "dns"), new SimpleCell("table02.col124.1", "dud"),
-            new BinaryCell("table02.col125.1", newBlob()))),
+            new BinaryCell("table02.col125.1", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff"))),
         new Row(3,
           Arrays.asList(new SimpleCell("table02.col121.2", "3"), new SimpleCell("table02.col122.2", "2"),
             new SimpleCell("table02.col123.2", "usl"), new SimpleCell("table02.col124.2", "aps"),
-            new BinaryCell("table02.col125.2", newBlob())))));
+            new BinaryCell("table02.col125.2", new TemporaryPathInputStreamProvider(newBlob()), "image/tiff")))));
     tableRows.put("schema02.table01", new ArrayList<Row>());
     tableRows.put("schema02.table02", new ArrayList<Row>());
 

dbptk-model/src/main/java/com/databasepreservation/DatabaseMigration.java

@@ -15,6 +15,7 @@
 
 import org.apache.commons.lang3.tuple.Pair;
 
+import com.databasepreservation.managers.ExportModuleContextManager;
 import com.databasepreservation.model.exception.ModuleException;
 import com.databasepreservation.model.modules.DatabaseImportModule;
 import com.databasepreservation.model.modules.DatabaseModuleFactory;
@@ -60,67 +61,75 @@ public DatabaseImportModule getImportModule() throws ModuleException {
   public void migrate() throws ModuleException {
     validate();
 
-    // get import module and export module instance
-    Map<Parameter, String> importParameters = buildParametersFromStringParameters(importModuleFactory,
-      importModuleFactoryStringParameters);
-    Map<Parameter, String> exportParameters = buildParametersFromStringParameters(exportModuleFactory,
-      exportModuleFactoryStringParameters);
-
-    DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter);
-    DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter);
-
-    List<DatabaseFilterModule> beforeFilterModules = new ArrayList<>();
-    List<DatabaseFilterModule> afterFilterModules = new ArrayList<>();
-    for (int i = 0; i < filterFactories.size(); i++) {
-      Map<Parameter, String> filterParameters = new HashMap<>();
-      if (!filterFactoriesStringParameters.isEmpty()) {
-        filterParameters = buildParametersFromStringParameters(filterFactories.get(i),
-          filterFactoriesStringParameters.get(i));
+    try {
+      if (exportModuleFactory != null) {
+        ExportModuleContextManager.getInstance().setup(exportModuleFactory);
       }
 
-      if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) {
-        afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter));
-      } else {
-        beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter));
+      // get import module and export module instance
+      Map<Parameter, String> importParameters = buildParametersFromStringParameters(importModuleFactory,
+        importModuleFactoryStringParameters);
+      Map<Parameter, String> exportParameters = buildParametersFromStringParameters(exportModuleFactory,
+        exportModuleFactoryStringParameters);
+
+      DatabaseImportModule importModule = importModuleFactory.buildImportModule(importParameters, reporter);
+      DatabaseFilterModule exportModule = exportModuleFactory.buildExportModule(exportParameters, reporter);
+
+      List<DatabaseFilterModule> beforeFilterModules = new ArrayList<>();
+      List<DatabaseFilterModule> afterFilterModules = new ArrayList<>();
+      for (int i = 0; i < filterFactories.size(); i++) {
+        Map<Parameter, String> filterParameters = new HashMap<>();
+        if (!filterFactoriesStringParameters.isEmpty()) {
+          filterParameters = buildParametersFromStringParameters(filterFactories.get(i),
+            filterFactoriesStringParameters.get(i));
+        }
+
+        if (filterFactories.get(i).getExecutionOrder().equals(ExecutionOrder.AFTER)) {
+          afterFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter));
+        } else {
+          beforeFilterModules.add(filterFactories.get(i).buildFilterModule(filterParameters, reporter));
+        }
       }
-    }
 
-    // set reporters
-    importModule.setOnceReporter(reporter);
-    for (DatabaseFilterModule filterModule : beforeFilterModules) {
-      filterModule.setOnceReporter(reporter);
-    }
+      // set reporters
+      importModule.setOnceReporter(reporter);
+      for (DatabaseFilterModule filterModule : beforeFilterModules) {
+        filterModule.setOnceReporter(reporter);
+      }
 
-    for (DatabaseFilterModule filterModule : afterFilterModules) {
-      filterModule.setOnceReporter(reporter);
-    }
-    for (DatabaseFilterModule filterModule : filterModules) {
-      filterModule.setOnceReporter(reporter);
-    }
-    exportModule.setOnceReporter(reporter);
+      for (DatabaseFilterModule filterModule : afterFilterModules) {
+        filterModule.setOnceReporter(reporter);
+      }
+      for (DatabaseFilterModule filterModule : filterModules) {
+        filterModule.setOnceReporter(reporter);
+      }
+      exportModule.setOnceReporter(reporter);
 
-    // create module chain with filters in the middle
-    Collections.reverse(filterModules);
-    Collections.reverse(beforeFilterModules);
-    Collections.reverse(afterFilterModules);
+      // create module chain with filters in the middle
+      Collections.reverse(filterModules);
+      Collections.reverse(beforeFilterModules);
+      Collections.reverse(afterFilterModules);
 
-    DatabaseFilterModule sinkModule = new SinkModule();
+      DatabaseFilterModule sinkModule = new SinkModule();
 
-    for (DatabaseFilterModule filterModule : afterFilterModules) {
-      sinkModule = filterModule.migrateDatabaseTo(sinkModule);
-    }
+      for (DatabaseFilterModule filterModule : afterFilterModules) {
+        sinkModule = filterModule.migrateDatabaseTo(sinkModule);
+      }
 
-    sinkModule = exportModule.migrateDatabaseTo(sinkModule);
+      sinkModule = exportModule.migrateDatabaseTo(sinkModule);
 
-    for (DatabaseFilterModule filterModule : beforeFilterModules) {
-      sinkModule = filterModule.migrateDatabaseTo(sinkModule);
-    }
+      for (DatabaseFilterModule filterModule : beforeFilterModules) {
+        sinkModule = filterModule.migrateDatabaseTo(sinkModule);
+      }
 
-    for (DatabaseFilterModule filterModule : filterModules) {
-      sinkModule = filterModule.migrateDatabaseTo(sinkModule);
-    }
+      for (DatabaseFilterModule filterModule : filterModules) {
+        sinkModule = filterModule.migrateDatabaseTo(sinkModule);
+      }
 
-    importModule.migrateDatabaseTo(sinkModule);
+      importModule.migrateDatabaseTo(sinkModule);
+    } finally {
+      ExportModuleContextManager.destroy();
+    }
   }
 
   /**

dbptk-model/src/main/java/com/databasepreservation/managers/ExportModuleContextManager.java

@@ -0,0 +1,43 @@
+package com.databasepreservation.managers;
+
+import com.databasepreservation.model.exception.UnsupportedModuleException;
+import com.databasepreservation.model.modules.DatabaseModuleFactory;
+import com.databasepreservation.model.parameters.Parameters;
+
+/**
+ * @author Gabriel Barros <gbarros@keep.pt>
+ */
+public class ExportModuleContextManager {
+  private static ExportModuleContextManager instance = null;
+  private String moduleName;
+  private Parameters exportModuleParameters;
+
+  public static ExportModuleContextManager getInstance() {
+    if (instance == null) {
+      instance = new ExportModuleContextManager();
+    }
+
+    return instance;
+  }
+
+  public static void destroy() {
+    instance = null;
+  }
+
+  public void setup(DatabaseModuleFactory exportModuleFactory) throws UnsupportedModuleException {
+    moduleName = exportModuleFactory.getModuleName();
+    exportModuleParameters = exportModuleFactory.getExportModuleParameters();
+  }
+
+  public String getModuleName() {
+    return moduleName;
+  }
+
+  public Parameters getExportModuleParameters() {
+    return exportModuleParameters;
+  }
+
+  public boolean isSiadDKModule() {
+    return moduleName.contains("siard-dk");
+  }
+}

dbptk-model/src/main/java/com/databasepreservation/model/data/BinaryCell.java

@@ -28,6 +28,7 @@ public class BinaryCell extends Cell implements InputStreamProvider {
   private InputStreamProvider inputStreamProvider;
   private String file;
   private long length;
+  private String mimeType;
 
   /**
    * Creates a binary cell. This binary cell will mostly just be a wrapper around
@@ -75,6 +76,12 @@ public BinaryCell(String id, InputStreamProvider inputStreamProvider) {
     this.inputStreamProvider = inputStreamProvider;
   }
 
+  public BinaryCell(String id, InputStreamProvider inputStreamProvider, String mimeType) {
+    super(id);
+    this.inputStreamProvider = inputStreamProvider;
+    this.mimeType = mimeType;
+  }
+
   /**
    * Creates a binary cell. This binary cell is a wrapper around a
    * ProvidesInputStream object (whilst also providing Cell functionality).
@@ -128,4 +135,12 @@ public String getFile() {
   public long getLength() {
     return length;
   }
+
+  public String getMimeType() {
+    return mimeType;
+  }
+
+  public void setFile(String file) {
+    this.file = file;
+  }
 }

dbptk-model/src/main/java/com/databasepreservation/model/parameters/Parameter.java

@@ -28,7 +28,7 @@ public enum INPUT_TYPE {
 
   /* GUI Helper for SIARD Export Module */
   public enum CATEGORY_TYPE {
-    SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, NONE
+    SIARD_EXPORT_OPTIONS, METADATA_EXPORT_OPTIONS, EXTERNAL_LOBS, CONVERSION_SERVICE_OPTIONS, NONE
   }
 
   public enum FILE_FILTER_TYPE {
@@ -312,7 +312,9 @@ public Parameter defaultSelectedIndex(Integer index) {
     return this;
   }
 
-  public Integer getDefaultSelectedIndex() { return defaultSelectedIndex; }
+  public Integer getDefaultSelectedIndex() {
+    return defaultSelectedIndex;
+  }
 
   /**
    * Gets the export option type for this parameter; Helper to automatize the

dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/CellHandlers/ExternalLOBSCellHandlerFileSystem.java

@@ -40,23 +40,25 @@ public Cell handleCell(String cellId, String cellValue) throws ModuleException {
       return newCell;
     }
 
-    Path blobPath = basePath.resolve(cellValue);
+    String cellValueStripTrailing = cellValue.stripTrailing();
+
+    Path blobPath = basePath.resolve(cellValueStripTrailing);
 
     if (Files.exists(blobPath)) {
       if (Files.isRegularFile(blobPath)) {
           try {
               newCell = new BinaryCell(cellId, new PathInputStreamProvider(blobPath));
           } catch (ModuleException e) {
               reporter.ignored("Cell " + cellId,
-                      blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValue);
+                      blobPath.toString() + " ignore due to: " + e.getMessage() + "; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing);
           }
       } else {
         reporter.ignored("Cell " + cellId,
-          blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValue);
+          blobPath.toString() + " is not a file; Base path: " + this.basePath + " Cell Value: " + cellValueStripTrailing);
       }
     } else {
       reporter.ignored("Cell " + cellId, "Path: " + blobPath.toString() + " could not be found; Base path: "
-        + this.basePath + " Cell Value: " + cellValue);
+        + this.basePath + " Cell Value: " + cellValueStripTrailing);
     }
     return newCell;
   }

dbptk-modules/dbptk-filter-external-lobs/src/main/java/com/databasepreservation/modules/externalLobs/ExternalLOBSFilter.java

@@ -14,10 +14,13 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.jetbrains.annotations.NotNull;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
+import com.databasepreservation.managers.ExportModuleContextManager;
 import com.databasepreservation.managers.ModuleConfigurationManager;
+import com.databasepreservation.model.data.BinaryCell;
 import com.databasepreservation.model.data.Cell;
 import com.databasepreservation.model.data.NullCell;
 import com.databasepreservation.model.data.Row;
@@ -93,12 +96,7 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException
 
             Type original = column.getType();
             description.append(". Original description: '").append(original.getDescription()).append("')");
-            SimpleTypeBinary newType = new SimpleTypeBinary();
-            newType.setSql99TypeName("BINARY VARYING", 1);
-            newType.setSql2008TypeName("BINARY VARYING", 1);
-            newType.setOriginalTypeName(original.getOriginalTypeName());
-            newType.setOutsideDatabase(true);
-
+            SimpleTypeBinary newType = getSimpleTypeBinary(original);
             column.setType(newType);
             column.setDescription(description.toString());
           }
@@ -110,6 +108,21 @@ public void handleStructure(DatabaseStructure structure) throws ModuleException
     this.exportModule.handleStructure(structure);
   }
 
+  @NotNull
+  private static SimpleTypeBinary getSimpleTypeBinary(Type original) {
+    SimpleTypeBinary newType = new SimpleTypeBinary();
+    if (ExportModuleContextManager.getInstance().isSiadDKModule()) {
+      newType.setSql99TypeName("BINARY LARGE OBJECT");
+      newType.setSql2008TypeName("BINARY LARGE OBJECT");
+    } else {
+      newType.setSql99TypeName("BINARY VARYING", 1);
+      newType.setSql2008TypeName("BINARY VARYING", 1);
+    }
+    newType.setOriginalTypeName(original.getOriginalTypeName());
+    newType.setOutsideDatabase(true);
+    return newType;
+  }
+
   @Override
   public void handleDataOpenSchema(String schemaName) throws ModuleException {
     this.exportModule.handleDataOpenSchema(schemaName);
@@ -155,6 +168,9 @@ public void handleDataRow(Row row) throws ModuleException {
               .get(currentTable.getId() + index);
             Cell newCell = getExternalLOBSCellHandler(externalLobsConfiguration).handleCell(cell.getId(),
               simpleCell.getSimpleData());
+            if (newCell instanceof BinaryCell binaryCell) {
+              binaryCell.setFile(simpleCell.getSimpleData());
+            }
             rowCells.set(index, newCell);
           } else {
             reporter.ignored("Cell " + cell.getId(), "reference to external LOB is null");

dbptk-modules/dbptk-module-jdbc/src/main/java/com/databasepreservation/modules/jdbc/out/JDBCExportModule.java

@@ -684,7 +684,7 @@ protected void handleSimpleTypeStringDataCell(String data, PreparedStatement ps,
 
   protected void handleSimpleTypeNumericExactDataCell(String data, PreparedStatement ps, int index, Cell cell,
     ColumnStructure column) throws SQLException {
-    if (data != null) {
+    if (data != null && !data.isEmpty()) {
       BigDecimal bd = new BigDecimal(data);
       ps.setBigDecimal(index, bd);
     } else {