diff --git a/docs/advanced_guidance/json_schemas/dataset.schema.json b/docs/advanced_guidance/json_schemas/dataset.schema.json index af8b620..909cbb1 100644 --- a/docs/advanced_guidance/json_schemas/dataset.schema.json +++ b/docs/advanced_guidance/json_schemas/dataset.schema.json @@ -13,6 +13,9 @@ }, "entity_relationships": { "$ref": "entity_relationships.schema.json" + }, + "error_report": { + "$ref": "error_report.schema.json" } }, "required": [ diff --git a/docs/advanced_guidance/json_schemas/error_report.schema.json b/docs/advanced_guidance/json_schemas/error_report.schema.json new file mode 100644 index 0000000..546d436 --- /dev/null +++ b/docs/advanced_guidance/json_schemas/error_report.schema.json @@ -0,0 +1,15 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "data-ingest:error_report.schema.json", + "title": "error_report", + "description": "Configuration for generating the error report.", + "type": "object", + "properties": { + "record_rejection_reporting": { + "description": "Whether to show/remove record rejection from the error report generated.", + "type": "boolean", + "default": true + } + }, + "additionalProperties": false +} diff --git a/src/dve/pipeline/pipeline.py b/src/dve/pipeline/pipeline.py index 8d9cfdb..4eba049 100644 --- a/src/dve/pipeline/pipeline.py +++ b/src/dve/pipeline/pipeline.py @@ -1013,6 +1013,7 @@ def error_report( if error_report_config.record_rejection_reporting or e != ErrorReportCategories.RECORD_REJECTION ], + error_report_config=error_report_config, ) workbook = er.ExcelFormat( diff --git a/src/dve/reporting/excel_report.py b/src/dve/reporting/excel_report.py index e0c4891..71eba72 100644 --- a/src/dve/reporting/excel_report.py +++ b/src/dve/reporting/excel_report.py @@ -16,6 +16,7 @@ from polars import DataFrame from polars.exceptions import ColumnNotFoundError +from dve.core_engine.configuration.v1 import V1ErrorReportConfig from dve.pipeline.utils import SubmissionStatus from dve.reporting.constants import ErrorReportCategories, ErrorReportStatus @@ -39,6 +40,8 @@ class SummaryItems: """List of aggregations to apply to the grouped up dataframe""" additional_columns: Optional[list] = None """any additional columns to add to the summary table""" + error_report_config: V1ErrorReportConfig = field(default_factory=V1ErrorReportConfig) + """user specificed config to change elements of the error report""" def create_summary_sheet( self, @@ -160,13 +163,15 @@ def _add_submission_info(self, status: str, summary: Worksheet): _records_rejected = self.submission_status.number_of_records else: _records_rejected = self.submission_status.number_of_records_rejected - summary.append( - [ - "", - "Total Number of Records Rejected", - _records_rejected, - ] - ) + + if self.error_report_config.record_rejection_reporting: + summary.append( + [ + "", + "Total Number of Records Rejected", + _records_rejected, + ] + ) summary.append(["", ""])