fix: total no records rejected showing when it shouldn't (#194)

Author: georgeRobertson

docs/advanced_guidance/json_schemas/dataset.schema.json

@@ -13,6 +13,9 @@
         },
         "entity_relationships": {
             "$ref": "entity_relationships.schema.json"
+        },
+        "error_report": {
+            "$ref": "error_report.schema.json"
         }
     },
     "required": [

docs/advanced_guidance/json_schemas/error_report.schema.json

@@ -0,0 +1,15 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "data-ingest:error_report.schema.json",
+    "title": "error_report",
+    "description": "Configuration for generating the error report.",
+    "type": "object",
+    "properties": {
+        "record_rejection_reporting": {
+            "description": "Whether to show/remove record rejection from the error report generated.",
+            "type": "boolean",
+            "default": true
+        }
+    },
+    "additionalProperties": false
+}

src/dve/pipeline/pipeline.py

@@ -1013,6 +1013,7 @@ def error_report(
                 if error_report_config.record_rejection_reporting
                 or e != ErrorReportCategories.RECORD_REJECTION
             ],
+            error_report_config=error_report_config,
         )
 
         workbook = er.ExcelFormat(

src/dve/reporting/excel_report.py

@@ -16,6 +16,7 @@
 from polars import DataFrame
 from polars.exceptions import ColumnNotFoundError
 
+from dve.core_engine.configuration.v1 import V1ErrorReportConfig
 from dve.pipeline.utils import SubmissionStatus
 from dve.reporting.constants import ErrorReportCategories, ErrorReportStatus
 
@@ -39,6 +40,8 @@ class SummaryItems:
     """List of aggregations to apply to the grouped up dataframe"""
     additional_columns: Optional[list] = None
     """any additional columns to add to the summary table"""
+    error_report_config: V1ErrorReportConfig = field(default_factory=V1ErrorReportConfig)
+    """user specificed config to change elements of the error report"""
 
     def create_summary_sheet(
         self,
@@ -160,13 +163,15 @@ def _add_submission_info(self, status: str, summary: Worksheet):
             _records_rejected = self.submission_status.number_of_records
         else:
             _records_rejected = self.submission_status.number_of_records_rejected
-        summary.append(
-            [
-                "",
-                "Total Number of Records Rejected",
-                _records_rejected,
-            ]
-        )
+
+        if self.error_report_config.record_rejection_reporting:
+            summary.append(
+                [
+                    "",
+                    "Total Number of Records Rejected",
+                    _records_rejected,
+                ]
+            )
 
         summary.append(["", ""])