From 7e3ed19c8957c46fd4ad7fd0d6923d417575359d Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 17 Oct 2026 14:41:56 +0700 Subject: [PATCH 01/15] fix(linux-import): keep an empty CSV field over the column default --- linux/crates/core/src/import/csv_import.rs | 22 ++++++++++++---- .../crates/drivers/sqlite/tests/csv_import.rs | 26 +++++++++++++++++++ 2 files changed, 43 insertions(+), 5 deletions(-) diff --git a/linux/crates/core/src/import/csv_import.rs b/linux/crates/core/src/import/csv_import.rs index e1dd3f3352..1679ac2776 100644 --- a/linux/crates/core/src/import/csv_import.rs +++ b/linux/crates/core/src/import/csv_import.rs @@ -177,16 +177,17 @@ pub fn row_to_cells( fn cell_for(text: &str, column: &ColumnInfo, options: &CsvImportOptions) -> Result { if text == options.null_marker { + // An empty string is a value in its own right for a text + // column, and the marker being empty is the common case. The + // file holds it, so it wins over the column's default. + if options.null_marker.is_empty() && column.column_type.kind.accepts_empty_string() { + return Ok(CellInput::Value(crate::value::Value::Text(String::new()))); + } // A NOT NULL column with a default is better served by its // default than by an insert the server will refuse. if !column.nullable && !column.default.is_none() { return Ok(CellInput::Default); } - // An empty string is a value in its own right for a text - // column, and the marker being empty is the common case. - if options.null_marker.is_empty() && column.column_type.kind.accepts_empty_string() { - return Ok(CellInput::Value(crate::value::Value::Text(String::new()))); - } return Ok(CellInput::Value(crate::value::Value::Null)); } parse_literal_text(text, &column.column_type) @@ -415,6 +416,17 @@ mod tests { assert_eq!(cells[0], CellInput::Default); } + #[test] + fn an_empty_field_for_a_not_null_text_column_with_a_default_stays_empty() { + let mut columns = vec![text_column("status")]; + columns[0].nullable = false; + columns[0].default = ColumnDefault::Literal(Value::Text("active".to_owned())); + + let cells = row_to_cells(&[String::new()], &[Some(0)], &columns, &options(), 2).expect("cells"); + + assert_eq!(cells[0], CellInput::Value(Value::Text(String::new()))); + } + #[test] fn a_field_the_column_cannot_hold_names_the_row_and_the_cell() { let columns = vec![int_column("id")]; diff --git a/linux/crates/drivers/sqlite/tests/csv_import.rs b/linux/crates/drivers/sqlite/tests/csv_import.rs index a04cd82728..8dc3e3cbd2 100644 --- a/linux/crates/drivers/sqlite/tests/csv_import.rs +++ b/linux/crates/drivers/sqlite/tests/csv_import.rs @@ -170,6 +170,32 @@ async fn a_column_the_file_does_not_name_keeps_its_default() { connection.close().await.expect("the close"); } +#[tokio::test] +async fn an_empty_field_is_kept_as_an_empty_string_over_the_default() { + let (_directory, connection) = open().await.expect("a database file"); + create( + connection.as_ref(), + "CREATE TABLE accounts (id INTEGER PRIMARY KEY, status TEXT NOT NULL DEFAULT 'active')", + ) + .await + .expect("the table"); + + import( + connection.as_ref(), + "accounts", + b"id,status\n1,\n", + &CsvImportOptions::default(), + ) + .await + .expect("the import"); + + let rows = query(connection.as_ref(), "SELECT status FROM accounts") + .await + .expect("the rows"); + assert_eq!(rows.rows[0][0], Value::Text(String::new())); + connection.close().await.expect("the close"); +} + #[tokio::test] async fn a_row_the_table_cannot_hold_leaves_the_table_as_it_was() { let (_directory, connection) = open().await.expect("a database file"); From f8f6b76d6ff8583b900bf2d9f381783f6ba4e26f Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 17 Oct 2026 16:23:52 +0700 Subject: [PATCH 02/15] fix(linux-import): keep the ids a CSV gives an identity column --- linux/crates/app/src/ui/import_dialog.rs | 71 +++++++++++---- linux/crates/app/src/ui/workspace/import.rs | 87 +++++++++++-------- linux/crates/app/src/ui/workspace/mod.rs | 2 +- linux/crates/core/src/ddl/identity_style.rs | 29 +++++++ linux/crates/core/src/dml/build_insert.rs | 13 +-- linux/crates/core/src/dml/tests.rs | 47 ++++++++-- linux/crates/core/src/import/csv_import.rs | 79 +++++++++++++---- linux/crates/core/src/import/import_column.rs | 69 +++++++++++++++ linux/crates/core/src/import/import_plan.rs | 30 +++++++ linux/crates/core/src/import/mod.rs | 4 + .../mssql/tests/integration/csv_import.rs | 36 ++++++++ .../drivers/mssql/tests/integration/main.rs | 1 + .../mysql/tests/integration/csv_import.rs | 35 ++++++++ .../drivers/mysql/tests/integration/main.rs | 1 + .../postgres/tests/integration/csv_import.rs | 50 +++++++++++ .../postgres/tests/integration/main.rs | 1 + .../crates/drivers/sqlite/tests/csv_import.rs | 56 ++++++++---- linux/crates/test-support/src/csv_import.rs | 49 +++++++++++ linux/crates/test-support/src/lib.rs | 2 + 19 files changed, 562 insertions(+), 100 deletions(-) create mode 100644 linux/crates/core/src/import/import_column.rs create mode 100644 linux/crates/core/src/import/import_plan.rs create mode 100644 linux/crates/drivers/mssql/tests/integration/csv_import.rs create mode 100644 linux/crates/drivers/mysql/tests/integration/csv_import.rs create mode 100644 linux/crates/drivers/postgres/tests/integration/csv_import.rs create mode 100644 linux/crates/test-support/src/csv_import.rs diff --git a/linux/crates/app/src/ui/import_dialog.rs b/linux/crates/app/src/ui/import_dialog.rs index c4a39f6b6e..7838f78874 100644 --- a/linux/crates/app/src/ui/import_dialog.rs +++ b/linux/crates/app/src/ui/import_dialog.rs @@ -5,9 +5,8 @@ use relm4::adw::prelude::*; use relm4::prelude::*; use relm4::{adw, gtk}; -use tablepro_core::ColumnInfo; use tablepro_core::export::CsvDelimiter; -use tablepro_core::import::{CsvImportOptions, CsvSheet, MAX_PREVIEW_ROWS, read_csv, suggest_mapping}; +use tablepro_core::import::{CsvImportOptions, CsvSheet, ImportColumn, MAX_PREVIEW_ROWS, read_csv, suggest_mapping}; use crate::runtime::DeliverInput; @@ -28,7 +27,7 @@ pub struct ImportCsvDialog { schema: Option, table: String, bytes: Vec, - columns: Vec, + columns: Vec, sheet: CsvSheet, options: CsvImportOptions, /// One row per table column, in column order. The selected index is @@ -45,7 +44,7 @@ pub struct ImportCsvDialog { pub struct ImportCsvDialogInit { pub schema: Option, pub table: String, - pub columns: Vec, + pub columns: Vec, pub bytes: Vec, } @@ -68,9 +67,9 @@ pub enum ImportCsvDialogInput { pub struct CsvImport { pub schema: Option, pub table: String, - pub columns: Vec, + pub columns: Vec, /// One entry per table column: the CSV field it takes, or `None` - /// for a column the user set to Skip. + /// for a column the user set to Skip or the server fills. pub mapping: Vec>, pub options: CsvImportOptions, pub bytes: Vec, @@ -266,8 +265,9 @@ impl ImportCsvDialog { for (column, field) in self.columns.iter().zip(&suggested) { let row = adw::ComboRow::builder() - .title(glib::markup_escape_text(&column.name)) + .title(glib::markup_escape_text(&column.info.name)) .model(>k::StringList::new(&choice_refs)) + .sensitive(!column.server_filled) .build(); row.set_selected(field.map_or(0, |index| index as u32 + 1)); let sender = sender.clone(); @@ -280,8 +280,10 @@ impl ImportCsvDialog { /// Show what the mapped field actually holds, under each column. fn show_samples(&self) { - for (row, field) in self.mapping_rows.borrow().iter().zip(self.mapping()) { + let rows = self.mapping_rows.borrow(); + for ((row, field), column) in rows.iter().zip(self.mapping()).zip(&self.columns) { row.set_subtitle(&match field { + _ if column.server_filled => crate::i18n::gettext("The database fills this column"), Some(index) => sample_text(&self.sheet, index, &self.options), None => crate::i18n::gettext("Uses the column's own default"), }); @@ -315,7 +317,9 @@ impl ImportCsvDialog { self.mapping_rows .borrow() .iter() - .map(|row| match row.selected() { + .zip(&self.columns) + .map(|(row, column)| match row.selected() { + _ if column.server_filled => None, 0 => None, index => Some(index as usize - 1), }) @@ -400,20 +404,24 @@ mod tests { #[cfg(test)] mod dialog_tests { use super::*; + use tablepro_core::ColumnInfo; use tablepro_core::column::{ColumnDefault, ColumnType}; /// The dialog only reads a column's name: the type decides what a /// value parses into, which happens after the dialog is gone. - fn column(name: &str) -> ColumnInfo { - ColumnInfo { - name: name.to_owned(), - column_type: ColumnType::unknown(), - nullable: true, - primary_key: false, - is_auto_increment: false, - is_generated: false, - default: ColumnDefault::None, - comment: None, + fn column(name: &str) -> ImportColumn { + ImportColumn { + info: ColumnInfo { + name: name.to_owned(), + column_type: ColumnType::unknown(), + nullable: true, + primary_key: false, + is_auto_increment: false, + is_generated: false, + default: ColumnDefault::None, + comment: None, + }, + server_filled: false, } } @@ -469,6 +477,31 @@ mod dialog_tests { assert_eq!(dialog.model().mapping(), vec![None, None, None]); } + #[gtk4::test] + fn a_column_the_server_fills_cannot_take_a_field() { + let mut id = column("id"); + id.server_filled = true; + let dialog = ImportCsvDialog::builder().launch(ImportCsvDialogInit { + schema: None, + table: "people".to_owned(), + columns: vec![id, column("name")], + bytes: b"id,name\n1,ada\n".to_vec(), + }); + crate::test_support::drain_main_context(); + + let rows = dialog.model().mapping_rows.borrow().clone(); + + assert_eq!(dialog.model().mapping(), vec![None, Some(1)]); + assert!( + !rows[0].is_sensitive(), + "a field could be picked for a column the server fills" + ); + assert_eq!( + rows[0].subtitle().unwrap_or_default().as_str(), + "The database fills this column" + ); + } + #[gtk4::test] fn a_file_that_is_not_csv_leaves_the_import_button_off() { let dialog = ImportCsvDialog::builder().launch(ImportCsvDialogInit { diff --git a/linux/crates/app/src/ui/workspace/import.rs b/linux/crates/app/src/ui/workspace/import.rs index 1c2cf195c9..8efe5b8009 100644 --- a/linux/crates/app/src/ui/workspace/import.rs +++ b/linux/crates/app/src/ui/workspace/import.rs @@ -5,11 +5,10 @@ use gtk4::{gio, glib}; use libadwaita::prelude::*; use relm4::{Component, ComponentController}; -use tablepro_core::dialect::SqlDialect; use tablepro_core::edit::CellInput; +use tablepro_core::import::{ImportColumn, plan_import}; use tablepro_core::meta::TableRef; -use tablepro_core::statement::GuardedStatement; -use tablepro_core::{ColumnInfo, ForeignKeyMode, WriteBatch, WriteKind, WriteReport}; +use tablepro_core::{WriteKind, WriteReport}; use tablepro_session::{OperationError, OperationKind, Session}; use crate::ui::file_read::{FileReadError, read_capped_file}; @@ -100,7 +99,7 @@ impl WorkspaceModel { &mut self, schema: Option, table: String, - columns: Vec, + columns: Vec, bytes: Vec, ) { use crate::ui::import_dialog::{ImportCsvDialog, ImportCsvDialogInit, ImportCsvDialogOutput}; @@ -194,11 +193,16 @@ async fn read_csv_file(file: &gio::File) -> Result, String> { }) } -async fn read_columns(session: Arc, table: TableRef) -> Result, OperationError> { +/// The table's columns with what the engine lets an insert write, so +/// the mapping never offers a column the server fills. +async fn read_columns(session: Arc, table: TableRef) -> Result, OperationError> { session .run(OperationKind::Metadata, move |lease, options| { let table = table.clone(); - async move { Ok(lease.fetch_columns(&table, options).await?) } + async move { + let definition = lease.fetch_table_definition(&table, options).await?; + Ok(definition.columns.iter().map(ImportColumn::from_definition).collect()) + } }) .await } @@ -208,7 +212,7 @@ async fn read_columns(session: Arc, table: TableRef) -> Result, + columns: Vec, rows: Vec>, ) -> Result<(Vec, WriteReport), OperationError> { let rows = Arc::new(rows); @@ -218,7 +222,7 @@ async fn write_import( let columns = columns.clone(); let rows = rows.clone(); async move { - let batch = import_batch(&*lease.dialect(), &table, &columns, &rows)?; + let batch = plan_import(&*lease.dialect(), &table, &columns, &rows)?; let sql: Vec = batch.steps.iter().map(|step| step.statement.sql().to_owned()).collect(); tracing::info!( rows = rows.len(), @@ -232,26 +236,6 @@ async fn write_import( .await } -/// One INSERT per row, as a single batch so the engine runs them in -/// one transaction where it has one. -fn import_batch( - dialect: &dyn SqlDialect, - table: &TableRef, - columns: &[ColumnInfo], - rows: &[Vec], -) -> Result { - let steps = rows - .iter() - .map(|cells| tablepro_core::dml::build_insert(dialect, table, columns, cells).map(GuardedStatement::unguarded)) - .collect::, _>>()?; - Ok(WriteBatch::new( - WriteKind::RowChanges, - table.clone(), - steps, - ForeignKeyMode::Enforced, - )) -} - /// Every row of the file as the cells to insert, or the first row that /// could not be read. /// @@ -294,7 +278,8 @@ fn import_rows(import: &CsvImport) -> Result>, String> { #[cfg(test)] mod tests { use tablepro_core::column::{ColumnDefault, ColumnKind, ColumnType, TextLength}; - use tablepro_core::{LivenessPolicy, Value}; + use tablepro_core::ddl::{ColumnAttributes, ColumnDefinition, IdentityStyle, TableDefinition}; + use tablepro_core::{ColumnInfo, LivenessPolicy, Value}; use tablepro_session::fakes::{FakeConnector, FakeCredentialSource}; use tablepro_session::operation::DefaultDeadlines; use tablepro_session::{SessionDeps, SessionHealth}; @@ -302,7 +287,7 @@ mod tests { use super::*; - fn column(name: &str) -> ColumnInfo { + fn info(name: &str) -> ColumnInfo { let mut column_type = ColumnType::untyped_text("text"); column_type.kind = ColumnKind::Text { length: TextLength::Unbounded, @@ -319,6 +304,23 @@ mod tests { } } + fn column(name: &str) -> ImportColumn { + ImportColumn { + info: info(name), + server_filled: false, + } + } + + fn definition(name: &str, identity: IdentityStyle) -> ColumnDefinition { + ColumnDefinition::new( + info(name), + ColumnAttributes { + identity, + ..ColumnAttributes::default() + }, + ) + } + fn session(script: ConnectionScript) -> (Arc, Arc) { let (pending, connection) = FakeConnector::pending_session(FakeConnector::request().unwrap(), script); let session = Session::start( @@ -348,9 +350,18 @@ mod tests { #[tokio::test(flavor = "multi_thread", worker_threads = 1)] async fn csv_import_reads_columns_through_session() { let mut script = ConnectionScript::default(); - script - .fetch_columns - .always(FakeReply::ok(vec![column("name"), column("city")])); + script.fetch_table_definition.always(FakeReply::ok(TableDefinition { + table: TableRef::unqualified("people"), + columns: vec![ + definition("id", IdentityStyle::MssqlIdentity), + definition("name", IdentityStyle::None), + ], + primary_key: None, + indexes: Vec::new(), + foreign_keys: Vec::new(), + inbound_foreign_keys: Vec::new(), + source: None, + })); let (session, connection) = session(script); let columns = crate::test_support::paused_tasks() @@ -359,8 +370,12 @@ mod tests { .unwrap() .unwrap(); - assert_eq!(columns.len(), 2); - assert_eq!(CallCounters::get(&connection.counters().fetch_columns), 1); + assert_eq!( + columns.iter().map(|column| column.server_filled).collect::>(), + vec![true, false], + "an identity that refuses a value was offered to the file" + ); + assert_eq!(CallCounters::get(&connection.counters().fetch_table_definition), 1); assert_eq!(*session.health().borrow(), SessionHealth::Online); } @@ -387,7 +402,7 @@ mod tests { #[test] fn a_bad_row_names_its_line_in_the_file() { let mut bad = import("name,city\nAda,London\n"); - bad.columns[1].column_type.kind = ColumnKind::Integer(tablepro_core::column::IntegerKind { + bad.columns[1].info.column_type.kind = ColumnKind::Integer(tablepro_core::column::IntegerKind { width: tablepro_core::column::IntegerWidth::W32, signed: true, }); diff --git a/linux/crates/app/src/ui/workspace/mod.rs b/linux/crates/app/src/ui/workspace/mod.rs index f39644dee9..63f8584233 100644 --- a/linux/crates/app/src/ui/workspace/mod.rs +++ b/linux/crates/app/src/ui/workspace/mod.rs @@ -401,7 +401,7 @@ pub(crate) enum WorkspaceInput { ImportCsvReady { schema: Option, table: String, - columns: Vec, + columns: Vec, bytes: Vec, }, ImportCsvRun(Box), diff --git a/linux/crates/core/src/ddl/identity_style.rs b/linux/crates/core/src/ddl/identity_style.rs index 9407cc1a6d..50f8d72ae3 100644 --- a/linux/crates/core/src/ddl/identity_style.rs +++ b/linux/crates/core/src/ddl/identity_style.rs @@ -22,3 +22,32 @@ pub enum IdentityStyle { /// SQL Server `IDENTITY(seed, increment)`. MssqlIdentity, } + +impl IdentityStyle { + /// Whether a plain INSERT may give the column a value of its own. + /// GENERATED ALWAYS needs `OVERRIDING SYSTEM VALUE` and SQL Server + /// IDENTITY needs `IDENTITY_INSERT`, so both refuse one. + pub fn accepts_explicit_value(self) -> bool { + !matches!(self, Self::GeneratedAlways | Self::MssqlIdentity) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn only_the_identities_that_need_an_override_refuse_a_value() { + assert!(!IdentityStyle::GeneratedAlways.accepts_explicit_value()); + assert!(!IdentityStyle::MssqlIdentity.accepts_explicit_value()); + for style in [ + IdentityStyle::None, + IdentityStyle::GeneratedByDefault, + IdentityStyle::SequenceDefault, + IdentityStyle::AutoIncrementKeyword, + IdentityStyle::SqliteRowidAlias, + ] { + assert!(style.accepts_explicit_value(), "{style:?}"); + } + } +} diff --git a/linux/crates/core/src/dml/build_insert.rs b/linux/crates/core/src/dml/build_insert.rs index 50ab06c608..415bfb65b7 100644 --- a/linux/crates/core/src/dml/build_insert.rs +++ b/linux/crates/core/src/dml/build_insert.rs @@ -6,11 +6,14 @@ use crate::statement::{Statement, StatementBuilder}; use super::BuildSqlError; -/// The INSERT for a draft row. +/// The INSERT for one row. /// -/// A cell the user left alone is left out of the statement, so the -/// server applies its own default. A generated or auto-increment -/// column is never written, because the server owns its value. +/// A `Default` cell is left out of the statement, so the server applies +/// its own default or next number. A generated column is never written, +/// because the server computes it. Every other value is written, an +/// auto-increment column's included: a value given for one is a key the +/// caller means to keep, and the caller leaves out the identities an +/// engine refuses a value for. pub fn build_insert( dialect: &dyn SqlDialect, table: &TableRef, @@ -32,7 +35,7 @@ pub fn build_insert( let Some(value) = cell.value() else { continue; }; - if column.is_generated || column.is_auto_increment { + if column.is_generated { continue; } markers.push(builder.bind(dialect, value.clone(), BindTarget::Column(column))?); diff --git a/linux/crates/core/src/dml/tests.rs b/linux/crates/core/src/dml/tests.rs index d2d2d9c5d7..93eab785f4 100644 --- a/linux/crates/core/src/dml/tests.rs +++ b/linux/crates/core/src/dml/tests.rs @@ -189,10 +189,9 @@ fn build_key_probe_stops_at_two_rows() { } #[test] -fn build_insert_omits_default_cells_and_generated_columns() { +fn build_insert_omits_generated_columns() { let dialect = TestDialect::exact(); let mut columns = columns(); - columns[0].is_auto_increment = true; columns[2].is_generated = true; let statement = build_insert( @@ -209,10 +208,48 @@ fn build_insert_omits_default_cells_and_generated_columns() { assert_eq!( statement.sql(), - r#"INSERT INTO "public"."users" ("name") VALUES ($1)"#, - "a column the server owns was written" + r#"INSERT INTO "public"."users" ("id", "name") VALUES ($1, $2)"#, + "a generated column was written" ); - assert_eq!(statement.params().len(), 1); + assert_eq!(statement.params().len(), 2); +} + +#[test] +fn build_insert_writes_the_value_given_for_an_auto_increment_column() { + let dialect = TestDialect::exact(); + let mut columns = columns(); + columns[0].is_auto_increment = true; + + let given = build_insert( + &dialect, + &table(), + &columns, + &[ + CellInput::Value(Value::Int(100)), + CellInput::Value(Value::Text("a".to_owned())), + CellInput::Default, + ], + ) + .expect("the insert"); + let left = build_insert( + &dialect, + &table(), + &columns, + &[ + CellInput::Default, + CellInput::Value(Value::Text("a".to_owned())), + CellInput::Default, + ], + ) + .expect("the insert"); + + assert_eq!( + given.sql(), + r#"INSERT INTO "public"."users" ("id", "name") VALUES ($1, $2)"#, + "the id the row was given was dropped" + ); + assert_eq!(given.params()[0].value(), &Value::Int(100)); + assert_eq!(left.sql(), r#"INSERT INTO "public"."users" ("name") VALUES ($1)"#); } #[test] diff --git a/linux/crates/core/src/import/csv_import.rs b/linux/crates/core/src/import/csv_import.rs index 1679ac2776..fceb660d6c 100644 --- a/linux/crates/core/src/import/csv_import.rs +++ b/linux/crates/core/src/import/csv_import.rs @@ -4,6 +4,8 @@ use crate::column::ColumnInfo; use crate::edit::{CellInput, parse_literal_text}; use crate::export::CsvDelimiter; +use super::ImportColumn; + /// How many rows the mapping dialog shows. Enough to tell whether the /// delimiter and the header switch are right, few enough that a /// gigabyte file still opens the dialog at once. @@ -134,12 +136,15 @@ pub fn read_csv(bytes: &[u8], options: &CsvImportOptions, limit: Option) /// /// Columns the file has no match for are left unmapped, which the /// dialog shows as "Skip" and the insert turns into the column's own -/// default. -pub fn suggest_mapping(headers: &[String], columns: &[ColumnInfo]) -> Vec> { +/// default. A column the server fills is never mapped. +pub fn suggest_mapping(headers: &[String], columns: &[ImportColumn]) -> Vec> { columns .iter() .map(|column| { - let wanted = column.name.trim().to_lowercase(); + if column.server_filled { + return None; + } + let wanted = column.info.name.trim().to_lowercase(); headers.iter().position(|header| header.trim().to_lowercase() == wanted) }) .collect() @@ -150,11 +155,12 @@ pub fn suggest_mapping(headers: &[String], columns: &[ColumnInfo]) -> Vec