From 307d8ff6a0a292c7b10813051e1b0d54129b9e49 Mon Sep 17 00:00:00 2001 From: Scott Severance Date: Fri, 18 Sep 2026 14:16:32 +0000 Subject: [PATCH 1/2] feat: add data validation and debug logging to _apply_loaded_df --- core/data_manager.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/core/data_manager.py b/core/data_manager.py index 5eb5ad2..855a1b3 100644 --- a/core/data_manager.py +++ b/core/data_manager.py @@ -1,12 +1,15 @@ """Data loading, export, and column management.""" import io +import logging import pandas as pd import numpy as np import streamlit as st from core.state import set_df, get_df, get_var_types, set_var_type from core import constants +logger = logging.getLogger(__name__) + def load_csv(uploaded_file): """Load data from a CSV file.""" @@ -47,6 +50,16 @@ def load_from_paste(text): def _apply_loaded_df(df): """Apply a loaded DataFrame to session state.""" + # Validate minimum data dimensions + if df.empty or len(df.columns) == 0: + logger.warning("Empty or malformed upload: shape=%s", df.shape) + raise ValueError("Uploaded data is empty or has no columns.") + + if len(df) == 0: + logger.warning("No data rows in upload, only columns: %s", list(df.columns)) + + logger.debug("Loaded DataFrame: shape=%s, columns=%s", df.shape, list(df.columns)) + # Pad with empty rows so user can add more data if len(df) < 20: extra = pd.DataFrame( @@ -65,6 +78,7 @@ def _auto_detect_type(col, df): """Auto-detect variable type for a column.""" series = df[col].dropna() if len(series) == 0: + logger.debug("Column '%s': empty, assigned METRIC", col) set_var_type(col, constants.METRIC) return @@ -75,13 +89,17 @@ def _auto_detect_type(col, df): if non_null_numeric / len(series) > 0.5: n_unique = numeric.dropna().nunique() if n_unique <= 2: + logger.debug("Column '%s': %d unique numeric values, assigned NOMINAL", col, n_unique) set_var_type(col, constants.NOMINAL) elif n_unique <= 7 and n_unique < len(series) * 0.3: + logger.debug("Column '%s': %d unique numeric values (sparse), assigned ORDINAL", col, n_unique) set_var_type(col, constants.ORDINAL) else: + logger.debug("Column '%s': %d unique numeric values, assigned METRIC", col, n_unique) set_var_type(col, constants.METRIC) else: # Non-numeric columns are Nominal regardless of cardinality. + logger.debug("Column '%s': non-numeric (%.1f%% coercible), assigned NOMINAL", col, 100 * non_null_numeric / len(series)) set_var_type(col, constants.NOMINAL) From 75e30921b875ae69ea96393b8901accf82ed57b7 Mon Sep 17 00:00:00 2001 From: "claude[bot]" <41898282+claude[bot]@users.noreply.github.com> Date: Fri, 18 Sep 2026 14:21:18 +0000 Subject: [PATCH 2/2] fix(review): correct header-only upload rejection and surface validation errors _apply_loaded_df rejected any zero-row DataFrame via df.empty, since DataFrame.empty is True whenever any axis has length 0 - this made the "only columns" warning branch dead code and broke header-only CSV/paste uploads that used to load fine and get padded for editing. Now only frames with zero columns are rejected. Also, the new ValueError from that validation was being caught by the callers' bare `except Exception` and replaced with a generic format error, so it never reached the user. load_csv/load_excel/load_from_paste now catch ValueError separately and return its message. Added regression tests for header-only uploads (still load and pad) and for the no-columns case (rejected, error message not swallowed). Co-Authored-By: Claude Sonnet 5 --- core/data_manager.py | 16 +++++++++++----- tests/test_data_manager.py | 29 ++++++++++++++++++++++++++++- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/core/data_manager.py b/core/data_manager.py index 855a1b3..a252e71 100644 --- a/core/data_manager.py +++ b/core/data_manager.py @@ -17,6 +17,8 @@ def load_csv(uploaded_file): df = pd.read_csv(uploaded_file, low_memory=False) _apply_loaded_df(df) return True, None + except ValueError as e: + return False, str(e) except Exception: return False, "Unable to load CSV file. Please check the format." @@ -27,6 +29,8 @@ def load_excel(uploaded_file): df = pd.read_excel(uploaded_file) _apply_loaded_df(df) return True, None + except ValueError as e: + return False, str(e) except Exception: return False, "Unable to load Excel file. Please check the format." @@ -44,6 +48,8 @@ def load_from_paste(text): df = pd.read_csv(io.StringIO(text), sep=";") _apply_loaded_df(df) return True, None + except ValueError as e: + return False, str(e) except Exception: return False, "Unable to parse pasted data. Please check the format." @@ -51,15 +57,15 @@ def load_from_paste(text): def _apply_loaded_df(df): """Apply a loaded DataFrame to session state.""" # Validate minimum data dimensions - if df.empty or len(df.columns) == 0: - logger.warning("Empty or malformed upload: shape=%s", df.shape) + if len(df.columns) == 0: + logger.warning("Malformed upload: shape=%s", df.shape) raise ValueError("Uploaded data is empty or has no columns.") - + if len(df) == 0: logger.warning("No data rows in upload, only columns: %s", list(df.columns)) - + logger.debug("Loaded DataFrame: shape=%s, columns=%s", df.shape, list(df.columns)) - + # Pad with empty rows so user can add more data if len(df) < 20: extra = pd.DataFrame( diff --git a/tests/test_data_manager.py b/tests/test_data_manager.py index 1c029b1..8b9d85d 100644 --- a/tests/test_data_manager.py +++ b/tests/test_data_manager.py @@ -7,7 +7,7 @@ import streamlit as st from core import constants -from core.data_manager import _auto_detect_type, add_column, load_csv +from core.data_manager import _apply_loaded_df, _auto_detect_type, add_column, load_csv @pytest.fixture(autouse=True) @@ -74,6 +74,33 @@ def test_load_csv_detects_every_column(): assert types["label"] == constants.NOMINAL +def test_load_csv_with_header_only_still_loads_and_pads_rows(): + # A CSV with columns but no data rows (e.g. a fresh export template) must + # still load so the user can type values into the padded editor rows. + csv = "num,grp,label\n" + ok, err = load_csv(io.StringIO(csv)) + + assert (ok, err) == (True, None) + df = st.session_state["df"] + assert list(df.columns) == ["num", "grp", "label"] + assert len(df) == 20 + + +def test_apply_loaded_df_rejects_dataframe_with_no_columns(): + with pytest.raises(ValueError): + _apply_loaded_df(pd.DataFrame()) + + +def test_load_csv_with_no_columns_reports_specific_error_not_generic_one(): + # A ValueError raised while applying the loaded frame must reach the + # caller verbatim instead of being swallowed by the catch-all Exception + # handler and replaced with the generic format-error message. + ok, err = load_csv(io.StringIO("")) + + assert ok is False + assert err != "Unable to load CSV file. Please check the format." + + def test_add_column_marks_the_new_column_as_metric(): add_column()