From 010357f45f8630733482892a86fe4f7559e60b9b Mon Sep 17 00:00:00 2001 From: satheesh142002 Date: Sun, 4 Oct 2026 11:15:54 +0530 Subject: [PATCH 1/3] feat(examples): add Streamlit cleaning app recipe --- examples/README.md | 1 + examples/integrations/streamlit_app.py | 112 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 examples/integrations/streamlit_app.py diff --git a/examples/README.md b/examples/README.md index f272de0..3ff5a91 100644 --- a/examples/README.md +++ b/examples/README.md @@ -38,6 +38,7 @@ Located in [`examples/integrations/`](integrations/): | **DuckDB** | [`duckdb_workflow.py`](integrations/duckdb_workflow.py) | Direct cleaning of `DuckDBPyRelation` objects and registering back to DuckDB for SQL. | | **scikit-learn** | [`sklearn_pipeline.py`](integrations/sklearn_pipeline.py) | Custom `FreshDataCleaner` transformer for native `sklearn.pipeline.Pipeline` usage. | | **FastAPI** | [`fastapi_clean_service.py`](integrations/fastapi_clean_service.py) | Asynchronous cleaning microservice accepting CSV uploads or JSON records without blocking the event loop. | +| **Streamlit** | [`streamlit_app.py`](integrations/streamlit_app.py) | Interactive CSV cleaning app with raw/cleaned previews, quality metrics, and a cleaned CSV download. | | **Apache Airflow** | [`airflow_task.py`](integrations/airflow_task.py) | Quality gating with `evaluate_trust_gate` and `FreshDataCleanOperator` DAG pattern. | | **Pandera** | [`09_pandera_recipe.py`](09_pandera_recipe.py) | Pre- and post-validation using declarative Pandera DataFrame schemas. | | **PyJanitor** | [`10_pyjanitor_interop.py`](10_pyjanitor_interop.py) | Combining explicit PyJanitor transforms with FreshData quality repair. | diff --git a/examples/integrations/streamlit_app.py b/examples/integrations/streamlit_app.py new file mode 100644 index 0000000..e3a01d1 --- /dev/null +++ b/examples/integrations/streamlit_app.py @@ -0,0 +1,112 @@ +"""Interactive Streamlit data-cleaning recipe. + +Install dependencies with:: + + pip install streamlit freshdata-cleaner + +Run the app with:: + + streamlit run examples/integrations/streamlit_app.py +""" + +from __future__ import annotations + +from io import BytesIO + +import pandas as pd +import streamlit as st + +import freshdata as fd + + +def demo_data() -> pd.DataFrame: + """Return a small messy dataset for trying the app without an upload.""" + return pd.DataFrame( + { + "Customer Name": [" Ava Chen ", "Noah Patel", "Mia Lopez", "Noah Patel"], + "Orders": ["3", "N/A", "5", "N/A"], + "Region": ["West", "East", "West", "East"], + } + ) + + +def load_uploaded_csv(contents: bytes) -> pd.DataFrame: + """Read uploaded CSV bytes into a DataFrame.""" + return pd.read_csv(BytesIO(contents)) + + +def count_modified_columns(report: object) -> int: + """Count columns named by applied cleaning actions and imputations.""" + modified = set(report.columns_imputed) + modified.update( + action.column + for action in report.actions + if action.column is not None and action.count > 0 + ) + return len(modified) + + +st.set_page_config(page_title="FreshData Cleaning Lab", layout="wide") +st.title("FreshData Cleaning Lab") +st.caption("Upload a CSV to compare its original data with FreshData's cleaned output.") + +uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"]) + +if uploaded_file is None: + raw_df = demo_data() + st.info("Showing sample data. Upload a CSV to clean your own dataset.") +else: + try: + raw_df = load_uploaded_csv(uploaded_file.getvalue()) + except (ValueError, UnicodeDecodeError, pd.errors.ParserError) as exc: + st.error(f"Could not read this CSV: {exc}") + st.stop() + +try: + cleaned_df, report = fd.clean(raw_df, return_report=True) +except Exception as exc: + st.error(f"FreshData could not clean this dataset: {exc}") + st.stop() + +st.subheader("Cleaning summary") +metric_columns = st.columns(4) +metric_columns[0].metric("Rows removed", max(report.rows_before - report.rows_after, 0)) +metric_columns[1].metric("Columns modified", count_modified_columns(report)) +imputed_values = sum( + action.count for action in report.actions if action.step == "impute" and action.count > 0 +) +metric_columns[2].metric("Missing values imputed", imputed_values) +metric_columns[3].metric("Duplicates dropped", report.duplicates_removed) + +raw_column, cleaned_column = st.columns(2) +with raw_column: + st.subheader("Raw data") + st.dataframe(raw_df, use_container_width=True) +with cleaned_column: + st.subheader("Cleaned data") + st.dataframe(cleaned_df, use_container_width=True) + +st.download_button( + "Download cleaned CSV", + data=cleaned_df.to_csv(index=False).encode("utf-8"), + file_name="cleaned_data.csv", + mime="text/csv", +) + +with st.expander("Quality report"): + st.write(report.summary()) + if report.actions: + st.dataframe( + pd.DataFrame( + [ + { + "Step": action.step, + "Column": action.column or "", + "Change": action.description, + "Count": action.count, + } + for action in report.actions + ] + ), + use_container_width=True, + ) From 840178367511e1726b77d38bc0824cef490b10ed Mon Sep 17 00:00:00 2001 From: satheesh142002 Date: Mon, 5 Oct 2026 22:01:59 +0530 Subject: [PATCH 2/3] fix(examples): count automatic missing value fills --- examples/integrations/streamlit_app.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/examples/integrations/streamlit_app.py b/examples/integrations/streamlit_app.py index e3a01d1..0d177ce 100644 --- a/examples/integrations/streamlit_app.py +++ b/examples/integrations/streamlit_app.py @@ -73,7 +73,11 @@ def count_modified_columns(report: object) -> int: metric_columns[0].metric("Rows removed", max(report.rows_before - report.rows_after, 0)) metric_columns[1].metric("Columns modified", count_modified_columns(report)) imputed_values = sum( - action.count for action in report.actions if action.step == "impute" and action.count > 0 + action.count + for action in report.actions + if action.step in {"impute", "missing"} + and action.description.startswith("filled ") + and action.count > 0 ) metric_columns[2].metric("Missing values imputed", imputed_values) metric_columns[3].metric("Duplicates dropped", report.duplicates_removed) From 8a2b78a0ed1f0fae912ea55f19766ef0df2232ea Mon Sep 17 00:00:00 2001 From: satheesh142002 Date: Tue, 6 Oct 2026 07:31:59 +0530 Subject: [PATCH 3/3] Fix Streamlit integration recipe metrics Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- examples/integrations/streamlit_app.py | 161 +++++++++++++++---------- 1 file changed, 95 insertions(+), 66 deletions(-) diff --git a/examples/integrations/streamlit_app.py b/examples/integrations/streamlit_app.py index 0d177ce..1db8b6d 100644 --- a/examples/integrations/streamlit_app.py +++ b/examples/integrations/streamlit_app.py @@ -12,6 +12,7 @@ from __future__ import annotations from io import BytesIO +from typing import Any import pandas as pd import streamlit as st @@ -35,82 +36,110 @@ def load_uploaded_csv(contents: bytes) -> pd.DataFrame: return pd.read_csv(BytesIO(contents)) -def count_modified_columns(report: object) -> int: - """Count columns named by applied cleaning actions and imputations.""" - modified = set(report.columns_imputed) +def count_modified_columns(report: Any) -> int: + """Count columns modified by applied cleaning actions, imputations, or drops.""" + modified = set(report.columns_imputed) | set(report.columns_dropped) modified.update( action.column for action in report.actions - if action.column is not None and action.count > 0 + if action.column is not None + and action.count > 0 + and getattr(action, "status", "automatic") in {"automatic", "approved"} ) return len(modified) -st.set_page_config(page_title="FreshData Cleaning Lab", layout="wide") -st.title("FreshData Cleaning Lab") -st.caption("Upload a CSV to compare its original data with FreshData's cleaned output.") +def main() -> None: + st.set_page_config(page_title="FreshData Cleaning Lab", layout="wide") + st.title("FreshData Cleaning Lab") + st.caption( + "Upload a CSV to compare its original data with FreshData's cleaned output." + ) -uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"]) + uploaded_file = st.file_uploader("Choose a CSV file", type=["csv"]) + + if uploaded_file is None: + raw_df = demo_data() + st.info("Showing sample data. Upload a CSV to clean your own dataset.") + else: + try: + raw_df = load_uploaded_csv(uploaded_file.getvalue()) + except (ValueError, UnicodeDecodeError, pd.errors.ParserError) as exc: + st.error(f"Could not read this CSV: {exc}") + st.stop() + + st.sidebar.header("Cleaning Settings") + strategy = st.sidebar.selectbox( + "Strategy", ["balanced", "conservative", "aggressive"], index=0 + ) + drop_duplicates = st.sidebar.checkbox("Drop duplicate rows", value=True) -if uploaded_file is None: - raw_df = demo_data() - st.info("Showing sample data. Upload a CSV to clean your own dataset.") -else: try: - raw_df = load_uploaded_csv(uploaded_file.getvalue()) - except (ValueError, UnicodeDecodeError, pd.errors.ParserError) as exc: - st.error(f"Could not read this CSV: {exc}") + cleaned_df, report = fd.clean( + raw_df, + strategy=strategy, + drop_duplicates=drop_duplicates, + return_report=True, + ) + except Exception as exc: + st.error(f"FreshData could not clean this dataset: {exc}") st.stop() -try: - cleaned_df, report = fd.clean(raw_df, return_report=True) -except Exception as exc: - st.error(f"FreshData could not clean this dataset: {exc}") - st.stop() - -st.subheader("Cleaning summary") -metric_columns = st.columns(4) -metric_columns[0].metric("Rows removed", max(report.rows_before - report.rows_after, 0)) -metric_columns[1].metric("Columns modified", count_modified_columns(report)) -imputed_values = sum( - action.count - for action in report.actions - if action.step in {"impute", "missing"} - and action.description.startswith("filled ") - and action.count > 0 -) -metric_columns[2].metric("Missing values imputed", imputed_values) -metric_columns[3].metric("Duplicates dropped", report.duplicates_removed) - -raw_column, cleaned_column = st.columns(2) -with raw_column: - st.subheader("Raw data") - st.dataframe(raw_df, use_container_width=True) -with cleaned_column: - st.subheader("Cleaned data") - st.dataframe(cleaned_df, use_container_width=True) - -st.download_button( - "Download cleaned CSV", - data=cleaned_df.to_csv(index=False).encode("utf-8"), - file_name="cleaned_data.csv", - mime="text/csv", -) - -with st.expander("Quality report"): - st.write(report.summary()) - if report.actions: - st.dataframe( - pd.DataFrame( - [ - { - "Step": action.step, - "Column": action.column or "", - "Change": action.description, - "Count": action.count, - } - for action in report.actions - ] - ), - use_container_width=True, + st.subheader("Cleaning summary") + metric_columns = st.columns(4) + metric_columns[0].metric( + "Rows removed", max(report.rows_before - report.rows_after, 0) + ) + metric_columns[1].metric("Columns modified", count_modified_columns(report)) + imputed_values = sum( + action.count + for action in report.actions + if ( + action.step == "impute" + or ( + action.step == "missing" + and action.column in report.columns_imputed + ) ) + and getattr(action, "status", "automatic") in {"automatic", "approved"} + and action.count > 0 + ) + metric_columns[2].metric("Missing values imputed", imputed_values) + metric_columns[3].metric("Duplicates dropped", report.duplicates_removed) + + raw_column, cleaned_column = st.columns(2) + with raw_column: + st.subheader("Raw data") + st.dataframe(raw_df, use_container_width=True) + with cleaned_column: + st.subheader("Cleaned data") + st.dataframe(cleaned_df, use_container_width=True) + + st.download_button( + "Download cleaned CSV", + data=cleaned_df.to_csv(index=False).encode("utf-8"), + file_name="cleaned_data.csv", + mime="text/csv", + ) + + with st.expander("Quality report"): + st.code(report.summary(), language="text") + if report.actions: + st.dataframe( + pd.DataFrame( + [ + { + "Step": action.step, + "Column": action.column or "", + "Change": action.description, + "Count": action.count, + } + for action in report.actions + ] + ), + use_container_width=True, + ) + + +if __name__ == "__main__": + main()