From ce3420bb06f007a3f96d06b1fb3801a4c4eabc1f Mon Sep 17 00:00:00 2001 From: ChudaykinAlex Date: Thu, 1 Oct 2026 14:51:09 +0300 Subject: [PATCH] Add data filters to gfix validation: -skip_data, -include_data, -skip_schema_data, -include_schema_data gfix -v -full and gfix -mend may fail to complete on a database with damaged records in unimportant tables. The new switches allow to exclude records of such tables from validation, so the rest of the database can still be checked and repaired. They are similar to the same gbak switches. - New gfix switches -skip_d(ata), -include(_data), -skip_schema_d(ata) and -include_schema_d(ata) accept a regular expression in SIMILAR TO syntax, matching is case insensitive. The switches require -full (or -mend), it's checked after parsing of all switches, so their order doesn't matter (also in services API). The pattern is checked when the switch is parsed. - Records of a table are skipped by the same rules as in BurpGlobals::skipRelation(): the table is filtered out either by the table name switches or by the schema name switches. The filters are not applied to system tables. - Records of a filtered relation, including large records and blobs, are not walked, so -mend doesn't change them: VDR_records is cleared for the relation in walk_relation() and its records are not passed to walk_record() and walk_blob(). Pointer, data and index pages of the relation are still walked as without -full. Orphan pages detection is disabled when records of some relation were actually skipped. - New DPB items isc_dpb_verify_skip_data, isc_dpb_verify_include_data, isc_dpb_verify_skip_schema_data, isc_dpb_verify_include_schema_data. gfix builds DPB with ClumpletReader::dpbList, so patterns longer than 255 bytes upgrade it to version 2. - New SPB items of isc_action_svc_repair isc_spb_rpr_skip_data, isc_spb_rpr_include_data, isc_spb_rpr_skip_schema_data, isc_spb_rpr_include_schema_data, fbsvcmgr options rpr_skip_data, rpr_include_data, rpr_skip_schema_data, rpr_include_schema_data. - gfix is linked with re2 (posix and MSVC builds). - Documentation: doc/README.gfix_data_filters.md. --- builds/posix/Makefile.in | 2 +- builds/win32/msvc15/gfix.vcxproj | 12 +-- doc/README.gfix_data_filters.md | 70 +++++++++++++++++ src/alice/alice.cpp | 70 +++++++++++++++++ src/alice/alice.h | 4 + src/alice/aliceswi.h | 29 ++++++- src/alice/exe.cpp | 18 ++++- src/common/IntlParametersBlock.cpp | 8 ++ src/common/classes/ClumpletReader.cpp | 4 + src/include/firebird/impl/consts_pub.h | 8 ++ src/include/firebird/impl/msg/gfix.h | 10 +++ src/include/gen/Firebird.pas | 8 ++ src/jrd/jrd.cpp | 19 ++++- src/jrd/svc.cpp | 12 +++ src/jrd/val_proto.h | 7 +- src/jrd/validation.cpp | 100 +++++++++++++++++++++++-- src/jrd/validation.h | 20 +++++ src/utilities/fbsvcmgr/fbsvcmgr.cpp | 4 + 18 files changed, 384 insertions(+), 21 deletions(-) create mode 100644 doc/README.gfix_data_filters.md diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index ab416b93ce5..34fa9757ab2 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -691,7 +691,7 @@ $(GSPLIT): $(GSPLIT_Objects) $(COMMON_LIB) gfix: $(GFIX) $(GFIX): $(GFIX_Objects) $(COMMON_LIB) - $(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) + $(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) $(RE2LIB) gsec: $(GSEC) diff --git a/builds/win32/msvc15/gfix.vcxproj b/builds/win32/msvc15/gfix.vcxproj index 6aa630b45d0..5bec957bd6b 100644 --- a/builds/win32/msvc15/gfix.vcxproj +++ b/builds/win32/msvc15/gfix.vcxproj @@ -163,7 +163,7 @@ EditAndContinue - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -175,7 +175,7 @@ WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -191,7 +191,7 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -207,7 +207,7 @@ WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -224,7 +224,7 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -239,7 +239,7 @@ WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/doc/README.gfix_data_filters.md b/doc/README.gfix_data_filters.md new file mode 100644 index 00000000000..2fa9b03ca20 --- /dev/null +++ b/doc/README.gfix_data_filters.md @@ -0,0 +1,70 @@ +# gfix: -SKIP_DATA / -INCLUDE_DATA / -SKIP_SCHEMA_DATA / -INCLUDE_SCHEMA_DATA + +Sometimes a damaged database can't be fully validated or repaired with `gfix -v -full` or `gfix -mend` +because of broken records in some unimportant tables. The switches below allow to exclude records of +such tables from validation, so the rest of the database can still be checked and repaired. + +The switches are similar to the same `gbak` switches and accept a regular expression in SQL syntax +(see `SIMILAR TO`). Matching is case insensitive. + +| Switch | Description | +|---------------------------|--------------------------------------------------------------------| +| `-SKIP_D(ATA)` | skip records validation of tables with matching names | +| `-INCLUDE(_DATA)` | validate records of tables with matching names only | +| `-SKIP_SCHEMA_D(ATA)` | skip records validation of tables in schemas with matching names | +| `-INCLUDE_SCHEMA_D(ATA)` | validate records of tables in schemas with matching names only | + +The rules to decide whether records of a table are validated are the same as in `gbak`: records are skipped +if the table is filtered out either by the table name switches or by the schema name switches. If a table +matches both `-SKIP_DATA` and `-INCLUDE_DATA` (or a schema matches both `-SKIP_SCHEMA_DATA` and +`-INCLUDE_SCHEMA_DATA`), its records are skipped. + +The filters are applied to user tables only, records of system tables are always validated. + +Records of skipped tables are not read at all, including big records (fragmented across several pages) +and blobs, so `-mend` doesn't change them. Pointer pages, data pages and index pages of skipped tables +are still walked and checked as without `-full`: the checks of full validation (record version chains, +index entries against records, consistency of index tree levels) are not performed for them. +Search for orphan pages is not performed when records of any table were skipped, as pages of big +records and blobs of skipped tables were not walked. + +The switches require `-validate` together with `-full` (or `-mend`, which implies both). + +Examples: +```shell +# Validate all records except records of table T1 of any schema +gfix -v -full -skip_data T1 database.fdb + +# Repair the database, skipping records of tables with names starting with LOG_ +gfix -mend -skip_data "LOG\_%" database.fdb + +# Validate records of tables of the schema S1 only +gfix -v -full -include_schema_data S1 database.fdb + +# Validate records of table T1 of the schema S1 only +gfix -v -full -include_schema_data S1 -include_data T1 database.fdb +``` + +## Services API + +The same filters are available in `isc_action_svc_repair` with the following string SPB items: + +| SPB item | fbsvcmgr option | Equivalent | +|-----------------------------------|----------------------------|-----------------------------| +| `isc_spb_rpr_skip_data` | `rpr_skip_data` | `gfix -skip_data` | +| `isc_spb_rpr_include_data` | `rpr_include_data` | `gfix -include_data` | +| `isc_spb_rpr_skip_schema_data` | `rpr_skip_schema_data` | `gfix -skip_schema_data` | +| `isc_spb_rpr_include_schema_data` | `rpr_include_schema_data` | `gfix -include_schema_data` | + +The items require `isc_spb_rpr_validate_db` and `isc_spb_rpr_full` (or `isc_spb_rpr_mend_db`) in `isc_spb_options`, +the order of the items in SPB doesn't matter. + +```shell +fbsvcmgr service_mgr action_repair dbname database.fdb rpr_validate_db rpr_full rpr_skip_data T1 +``` + +## DPB + +`gfix` passes the filters to the engine using the following string DPB items, used together with `isc_dpb_verify`: +`isc_dpb_verify_skip_data`, `isc_dpb_verify_include_data`, `isc_dpb_verify_skip_schema_data`, +`isc_dpb_verify_include_schema_data`. diff --git a/src/alice/alice.cpp b/src/alice/alice.cpp index 06b1ce4357a..a7cd9fa5744 100644 --- a/src/alice/alice.cpp +++ b/src/alice/alice.cpp @@ -50,6 +50,8 @@ #include "../common/utils_proto.h" #include "../common/classes/Switches.h" #include "../common/SimpleStatusVector.h" +#include "../common/SimilarToRegex.h" +#include "../common/isc_f_proto.h" #include "../alice/aliceswi.h" #ifdef HAVE_UNISTD_H @@ -95,6 +97,37 @@ constexpr int ALICE_MSG_FAC = FB_IMPL_MSG_FACILITY_GFIX; static void alice_output(bool error, const SCHAR*, ...) ATTRIBUTE_FORMAT(2,3); +//____________________________________________________________ +// +// Get regular expression argument of -skip_data / -include_data +// and similar switches and check it can be compiled. +// + +static const char* get_data_filter(Firebird::UtilSvc* uSvc, const char**& argv, int& argc, + USHORT missingMsg) +{ + if (--argc <= 0) + ALICE_error(missingMsg); + + const char* const pattern = *argv++; + + try + { + Firebird::string filter(pattern); + if (!uSvc->utf8FileNames()) + ISC_systemToUtf8(filter); + + Firebird::SimilarToRegex matcher(*getDefaultMemoryPool(), + Firebird::SimilarToFlag::CASE_INSENSITIVE, + filter.c_str(), filter.length(), "\\", 1); + } + catch (const Firebird::Exception&) + { + ALICE_error(146, SafeArg() << pattern); // msg 146: invalid regular expression "@1" + } + + return pattern; +} //____________________________________________________________ @@ -143,6 +176,10 @@ int alice(Firebird::UtilSvc* uSvc) tdgbl->ALICE_data.ua_user = NULL; tdgbl->ALICE_data.ua_role = NULL; tdgbl->ALICE_data.ua_password = NULL; + tdgbl->ALICE_data.ua_skip_data = NULL; + tdgbl->ALICE_data.ua_include_data = NULL; + tdgbl->ALICE_data.ua_skip_schema_data = NULL; + tdgbl->ALICE_data.ua_include_schema_data = NULL; #ifdef TRUSTED_AUTH tdgbl->ALICE_data.ua_trusted = false; #endif @@ -477,6 +514,30 @@ int alice(Firebird::UtilSvc* uSvc) else ALICE_error(135); // msg 135: replica mode (none / read_only / read_write) required } + + if (table->in_sw_value & sw_skip_data) + { + tdgbl->ALICE_data.ua_skip_data = get_data_filter(uSvc, argv, argc, 142); + // msg 142: missing regular expression to skip tables + } + + if (table->in_sw_value & sw_include_data) + { + tdgbl->ALICE_data.ua_include_data = get_data_filter(uSvc, argv, argc, 143); + // msg 143: missing regular expression to include tables + } + + if (table->in_sw_value & sw_skip_schema_data) + { + tdgbl->ALICE_data.ua_skip_schema_data = get_data_filter(uSvc, argv, argc, 144); + // msg 144: missing regular expression to skip schemas + } + + if (table->in_sw_value & sw_include_schema_data) + { + tdgbl->ALICE_data.ua_include_schema_data = get_data_filter(uSvc, argv, argc, 145); + // msg 145: missing regular expression to include schemas + } } // put this here since to put it above overly complicates the parsing. @@ -523,6 +584,15 @@ int alice(Firebird::UtilSvc* uSvc) ALICE_exit(FINI_ERROR, tdgbl); } + // Data filters affect records validation only, so they are useless without -full. + // Can't use in_sw_requires since it only looks backwards on command line, + // and services API puts switches on command line in the order of SPB items. + if ((flags & sw_data_filters) && !(flags & sw_full)) + { + ALICE_error(147); + // msg 147: -skip_data, -include_data, -skip_schema_data and -include_schema_data require -full + } + if (!database) { ALICE_error(23); // msg 23: please retry, giving a database name } diff --git a/src/alice/alice.h b/src/alice/alice.h index f2527d6d64d..3bf00b5e7c0 100644 --- a/src/alice/alice.h +++ b/src/alice/alice.h @@ -93,6 +93,10 @@ struct user_action alice_shut_mode ua_shutdown_mode; alice_repl_mode ua_replica_mode; SSHORT ua_parallel_workers; + const char* ua_skip_data; + const char* ua_include_data; + const char* ua_skip_schema_data; + const char* ua_include_schema_data; }; diff --git a/src/alice/aliceswi.h b/src/alice/aliceswi.h index 92b8f373d31..9a7b78e4ace 100644 --- a/src/alice/aliceswi.h +++ b/src/alice/aliceswi.h @@ -67,10 +67,14 @@ inline constexpr SINT64 sw_icu = QUADCONST(0x0000002000000000); inline constexpr SINT64 sw_role = QUADCONST(0x0000004000000000); inline constexpr SINT64 sw_replica = QUADCONST(0x0000008000000000); inline constexpr SINT64 sw_upgrade = QUADCONST(0x0000010000000000); +inline constexpr SINT64 sw_skip_data = QUADCONST(0x0000020000000000); +inline constexpr SINT64 sw_include_data = QUADCONST(0x0000040000000000); +inline constexpr SINT64 sw_skip_schema_data = QUADCONST(0x0000080000000000); +inline constexpr SINT64 sw_include_schema_data = QUADCONST(0x0000100000000000); // Popular combination of compatible switches inline constexpr SINT64 sw_auth_set = sw_user | sw_password | sw_role | sw_fetch_password | sw_trusted_auth; - +inline constexpr SINT64 sw_data_filters = sw_skip_data | sw_include_data | sw_skip_schema_data | sw_include_schema_data; enum alice_switches { @@ -128,7 +132,11 @@ enum alice_switches IN_SW_ALICE_ROLE = 49, IN_SW_ALICE_REPLICA = 50, IN_SW_ALICE_PARALLEL_WORKERS = 51, - IN_SW_ALICE_UPGRADE = 52 + IN_SW_ALICE_UPGRADE = 52, + IN_SW_ALICE_SKIP_DATA = 53, + IN_SW_ALICE_INCLUDE_DATA = 54, + IN_SW_ALICE_SKIP_SCHEMA_DATA = 55, + IN_SW_ALICE_INCLUDE_SCHEMA_DATA = 56 }; static inline constexpr const char* ALICE_SW_ASYNC = "ASYNC"; @@ -190,6 +198,12 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = {IN_SW_ALICE_ICU, isc_spb_rpr_icu, "ICU", sw_icu, 0, sw_shut, false, true, 131, 3, NULL}, // msg 131: \t-icu\t\tfix database to be usable with present ICU version + {IN_SW_ALICE_INCLUDE_DATA, isc_spb_rpr_include_data, "INCLUDE_DATA", sw_include_data, + 0, sw_include_data, false, false, 139, 7, NULL}, + // msg 139: -include(_data) validate records of matching tables only (-full) + {IN_SW_ALICE_INCLUDE_SCHEMA_DATA, isc_spb_rpr_include_schema_data, "INCLUDE_SCHEMA_DATA", sw_include_schema_data, + 0, sw_include_schema_data, false, false, 141, 16, NULL}, + // msg 141: -include_schema_d(ata) validate records of tables in matching schemas only (-full) {IN_SW_ALICE_KILL, isc_spb_rpr_kill_shadows, "KILL_SHADOW", sw_kill, 0, 0, false, true, 36, 1, NULL}, // msg 36: \t-kill\t\tkill all unavailable shadow files @@ -197,7 +211,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = 0, ~(sw_list | sw_auth_set | sw_nolinger), false, true, 37, 1, NULL}, // msg 37: \t-list\t\tshow limbo transactions {IN_SW_ALICE_MEND, isc_spb_rpr_mend_db, "MEND", sw_mend | sw_validate | sw_full, - 0, ~(sw_no_update | sw_auth_set | sw_nolinger), false, true, 38, 2, NULL}, + 0, ~(sw_no_update | sw_auth_set | sw_nolinger | sw_data_filters), false, true, 38, 2, NULL}, // msg 38: \t-mend\t\tprepare corrupt database for backup {IN_SW_ALICE_MODE, 0, "MODE", sw_mode, 0, ~(sw_mode | sw_auth_set | sw_nolinger), false, false, 109, 2, NULL}, @@ -240,6 +254,12 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = {IN_SW_ALICE_SET_DB_SQL_DIALECT, isc_spb_prp_set_sql_dialect, "SQL_DIALECT", sw_set_db_dialect, 0, 0, false, false, 111, 2, NULL}, // msg 111: \t-SQL_dialect\t\set dataabse dialect n + {IN_SW_ALICE_SKIP_DATA, isc_spb_rpr_skip_data, "SKIP_DATA", sw_skip_data, + 0, sw_skip_data, false, false, 138, 6, NULL}, + // msg 138: -skip_d(ata) skip records validation of matching tables (-full) + {IN_SW_ALICE_SKIP_SCHEMA_DATA, isc_spb_rpr_skip_schema_data, "SKIP_SCHEMA_DATA", sw_skip_schema_data, + 0, sw_skip_schema_data, false, false, 140, 13, NULL}, + // msg 140: -skip_schema_d(ata) skip records validation of tables in matching schemas (-full) {IN_SW_ALICE_SWEEP, isc_spb_rpr_sweep_db, "SWEEP", sw_sweep, 0, ~(sw_sweep | sw_auth_set | sw_nolinger), false, true, 45, 2, NULL}, // msg 45: \t-sweep\t\tforce garbage collection @@ -268,7 +288,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = 0, sw_trusted_auth, false, false, 50, 4, NULL}, // msg 50: \t-user\t\tdefault user name {IN_SW_ALICE_VALIDATE, isc_spb_rpr_validate_db, "VALIDATE", sw_validate, - 0, ~(sw_validate | sw_auth_set | sw_nolinger), false, true, 51, 1, NULL}, + 0, ~(sw_validate | sw_auth_set | sw_nolinger | sw_data_filters), false, true, 51, 1, NULL}, // msg 51: \t-validate\tvalidate database structure {IN_SW_ALICE_WRITE, 0, "WRITE", sw_write, 0, ~(sw_write | sw_auth_set | sw_nolinger), false, false, 52, 1, NULL}, @@ -280,6 +300,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = #endif {IN_SW_ALICE_VERSION, 0, "Z", 0, 0, 0, false, false, 54, 1, NULL}, + // msg 54: \t-z\t\tprint software version number /************************************************************************/ // WARNING: All new switches should be added right before this comments diff --git a/src/alice/exe.cpp b/src/alice/exe.cpp index 4cdfa8da4a3..32a3a16c5df 100644 --- a/src/alice/exe.cpp +++ b/src/alice/exe.cpp @@ -83,7 +83,7 @@ int EXE_action(const TEXT* database, const SINT64 switches) // generate the database parameter block for the attach, // based on the various switches - Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::Tagged, MAX_DPB_SIZE); + Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::dpbList, MAX_DPB_SIZE); buildDpb(dpb, switches); FB_API_HANDLE handle = 0; @@ -159,7 +159,7 @@ int EXE_two_phase(const TEXT* database, const SINT64 switches) // generate the database parameter block for the attach, // based on the various switches - Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::Tagged, MAX_DPB_SIZE); + Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::dpbList, MAX_DPB_SIZE); buildDpb(dpb, switches); FB_API_HANDLE handle = 0; @@ -231,6 +231,20 @@ static void buildDpb(Firebird::ClumpletWriter& dpb, const SINT64 switches) if (switches & sw_ignore) b |= isc_dpb_ignore; dpb.insertByte(isc_dpb_verify, b); + + const user_action& ua = tdgbl->ALICE_data; + + if (switches & sw_skip_data) + dpb.insertString(isc_dpb_verify_skip_data, ua.ua_skip_data); + + if (switches & sw_include_data) + dpb.insertString(isc_dpb_verify_include_data, ua.ua_include_data); + + if (switches & sw_skip_schema_data) + dpb.insertString(isc_dpb_verify_skip_schema_data, ua.ua_skip_schema_data); + + if (switches & sw_include_schema_data) + dpb.insertString(isc_dpb_verify_include_schema_data, ua.ua_include_schema_data); } else if (switches & sw_housekeeping) { dpb.insertInt(isc_dpb_sweep_interval, tdgbl->ALICE_data.ua_sweep_interval); diff --git a/src/common/IntlParametersBlock.cpp b/src/common/IntlParametersBlock.cpp index 7830b40a181..01447dc6da3 100644 --- a/src/common/IntlParametersBlock.cpp +++ b/src/common/IntlParametersBlock.cpp @@ -196,6 +196,10 @@ IntlParametersBlock::TagType IntlDpb::checkTag(UCHAR tag, const char** tagName) FB_IPB_TAG(isc_dpb_os_user); FB_IPB_TAG(isc_dpb_owner); FB_IPB_TAG(isc_dpb_search_path); + FB_IPB_TAG(isc_dpb_verify_skip_data); + FB_IPB_TAG(isc_dpb_verify_include_data); + FB_IPB_TAG(isc_dpb_verify_skip_schema_data); + FB_IPB_TAG(isc_dpb_verify_include_schema_data); return TAG_STRING; default: return TAG_SKIP; @@ -276,6 +280,10 @@ IntlParametersBlock::TagType IntlSpbStart::checkTag(UCHAR tag, const char** tagN switch (tag) { FB_IPB_TAG(isc_spb_tra_db_path); + FB_IPB_TAG(isc_spb_rpr_skip_data); + FB_IPB_TAG(isc_spb_rpr_include_data); + FB_IPB_TAG(isc_spb_rpr_skip_schema_data); + FB_IPB_TAG(isc_spb_rpr_include_schema_data); return TAG_STRING; } break; diff --git a/src/common/classes/ClumpletReader.cpp b/src/common/classes/ClumpletReader.cpp index 32f24050c63..e1a246cd636 100644 --- a/src/common/classes/ClumpletReader.cpp +++ b/src/common/classes/ClumpletReader.cpp @@ -346,6 +346,10 @@ ClumpletReader::ClumpletType ClumpletReader::getClumpletType(UCHAR tag) const switch (tag) { case isc_spb_dbname: + case isc_spb_rpr_skip_data: + case isc_spb_rpr_include_data: + case isc_spb_rpr_skip_schema_data: + case isc_spb_rpr_include_schema_data: return StringSpb; case isc_spb_options: case isc_spb_rpr_commit_trans: diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index 3831a26788a..f7c73fef506 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -138,6 +138,10 @@ #define isc_dpb_search_path 105 #define isc_dpb_blr_request_search_path 106 #define isc_dpb_gbak_restore_has_schema 107 +#define isc_dpb_verify_skip_data 108 +#define isc_dpb_verify_include_data 109 +#define isc_dpb_verify_skip_schema_data 110 +#define isc_dpb_verify_include_schema_data 111 /**************************************************/ @@ -537,6 +541,10 @@ #define isc_spb_rpr_rollback_trans_64 50 #define isc_spb_rpr_recover_two_phase_64 51 #define isc_spb_rpr_par_workers 52 +#define isc_spb_rpr_skip_data 53 +#define isc_spb_rpr_include_data 54 +#define isc_spb_rpr_skip_schema_data 55 +#define isc_spb_rpr_include_schema_data 56 #define isc_spb_rpr_validate_db 0x01 #define isc_spb_rpr_sweep_db 0x02 diff --git a/src/include/firebird/impl/msg/gfix.h b/src/include/firebird/impl/msg/gfix.h index f8f5135e0da..e277a762f5c 100644 --- a/src/include/firebird/impl/msg/gfix.h +++ b/src/include/firebird/impl/msg/gfix.h @@ -135,3 +135,13 @@ FB_IMPL_MSG_SYMBOL(GFIX, 134, gfix_opt_repl, " -repl(ica) replica mo FB_IMPL_MSG_SYMBOL(GFIX, 135, gfix_repl_mode_req, "replica mode (none / read_only / read_write) required") FB_IMPL_MSG_SYMBOL(GFIX, 136, gfix_opt_parallel, " -par(allel) parallel workers (-sweep, -icu)") FB_IMPL_MSG_SYMBOL(GFIX, 137, gfix_opt_upgrade, " -up(grade) upgrade database ODS") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 138, " -skip_d(ata) skip records validation of matching tables (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 139, " -include(_data) validate records of matching tables only (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 140, " -skip_schema_d(ata) skip records validation of tables in matching schemas (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 141, " -include_schema_d(ata) validate records of tables in matching schemas only (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 142, "missing regular expression to skip tables") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 143, "missing regular expression to include tables") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 144, "missing regular expression to skip schemas") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 145, "missing regular expression to include schemas") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 146, "invalid regular expression \"@1\"") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 147, "-skip_data, -include_data, -skip_schema_data and -include_schema_data require -full") diff --git a/src/include/gen/Firebird.pas b/src/include/gen/Firebird.pas index 393399381a7..d1a16e25912 100644 --- a/src/include/gen/Firebird.pas +++ b/src/include/gen/Firebird.pas @@ -4341,6 +4341,10 @@ IPerformanceStatsImpl = class(IPerformanceStats) isc_dpb_search_path = byte(105); isc_dpb_blr_request_search_path = byte(106); isc_dpb_gbak_restore_has_schema = byte(107); + isc_dpb_verify_skip_data = byte(108); + isc_dpb_verify_include_data = byte(109); + isc_dpb_verify_skip_schema_data = byte(110); + isc_dpb_verify_include_schema_data = byte(111); isc_dpb_address = byte(1); isc_dpb_addr_protocol = byte(1); isc_dpb_addr_endpoint = byte(2); @@ -4570,6 +4574,10 @@ IPerformanceStatsImpl = class(IPerformanceStats) isc_spb_rpr_rollback_trans_64 = byte(50); isc_spb_rpr_recover_two_phase_64 = byte(51); isc_spb_rpr_par_workers = byte(52); + isc_spb_rpr_skip_data = byte(53); + isc_spb_rpr_include_data = byte(54); + isc_spb_rpr_skip_schema_data = byte(55); + isc_spb_rpr_include_schema_data = byte(56); isc_spb_rpr_validate_db = $01; isc_spb_rpr_sweep_db = $02; isc_spb_rpr_mend_db = $04; diff --git a/src/jrd/jrd.cpp b/src/jrd/jrd.cpp index fdb63403818..6efd2a984db 100644 --- a/src/jrd/jrd.cpp +++ b/src/jrd/jrd.cpp @@ -1069,6 +1069,7 @@ namespace Jrd string dpb_decfloat_round; string dpb_decfloat_traps; string dpb_owner; + ValidationDataFilters dpb_verify_filters; Firebird::ObjectsArray dpb_schema_search_path; Firebird::ObjectsArray dpb_blr_request_schema_search_path; @@ -2096,7 +2097,7 @@ JAttachment* JProvider::internalAttach(CheckStatusWrapper* user_status, const ch AutoSetRestoreFlag noCleanup(&attachment->att_flags, ATT_no_cleanup, true); VIO_fini(tdbb); - if (!VAL_validate(tdbb, options.dpb_verify)) + if (!VAL_validate(tdbb, options.dpb_verify, options.dpb_verify_filters)) ERR_punt(); } @@ -7050,6 +7051,22 @@ void DatabaseOptions::get(const UCHAR* dpb, FB_SIZE_T dpb_length, bool& invalid_ dpb_flags |= DBB_damaged; break; + case isc_dpb_verify_skip_data: + getString(rdr, dpb_verify_filters.skipData); + break; + + case isc_dpb_verify_include_data: + getString(rdr, dpb_verify_filters.includeData); + break; + + case isc_dpb_verify_skip_schema_data: + getString(rdr, dpb_verify_filters.skipSchemaData); + break; + + case isc_dpb_verify_include_schema_data: + getString(rdr, dpb_verify_filters.includeSchemaData); + break; + case isc_dpb_trace: rdr.getInt(); break; diff --git a/src/jrd/svc.cpp b/src/jrd/svc.cpp index f08a0b8024f..c12b781d249 100644 --- a/src/jrd/svc.cpp +++ b/src/jrd/svc.cpp @@ -3047,6 +3047,18 @@ bool Service::process_switches(ClumpletReader& spb, string& switches) } get_action_svc_data(spb, switches, bigint); break; + + case isc_spb_rpr_skip_data: + case isc_spb_rpr_include_data: + case isc_spb_rpr_skip_schema_data: + case isc_spb_rpr_include_schema_data: + if (!get_action_svc_parameter(spb.getClumpTag(), alice_in_sw_table, switches)) + { + return false; + } + get_action_svc_string(spb, switches); + break; + case isc_spb_prp_write_mode: case isc_spb_prp_access_mode: case isc_spb_prp_reserve_space: diff --git a/src/jrd/val_proto.h b/src/jrd/val_proto.h index 1272059dd14..dc33300c812 100644 --- a/src/jrd/val_proto.h +++ b/src/jrd/val_proto.h @@ -24,7 +24,12 @@ #ifndef JRD_VAL_PROTO_H #define JRD_VAL_PROTO_H -bool VAL_validate(Jrd::thread_db*, USHORT); +namespace Jrd +{ + struct ValidationDataFilters; +} + +bool VAL_validate(Jrd::thread_db*, USHORT, const Jrd::ValidationDataFilters&); int VAL_service(Firebird::UtilSvc*); inline constexpr int IN_SW_VAL_TAB_INCL = 1; diff --git a/src/jrd/validation.cpp b/src/jrd/validation.cpp index b020cc5b3cc..a3aafa42f30 100644 --- a/src/jrd/validation.cpp +++ b/src/jrd/validation.cpp @@ -592,7 +592,7 @@ static void print_rhd(USHORT, const rhd*); #endif -static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern) +static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern, unsigned flags = 0) { SimilarToRegex* matcher = NULL; try @@ -604,7 +604,7 @@ static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern //// TODO: Should this be different than trace and replication //// and use case sensitive matcher? matcher = FB_NEW_POOL(*tdbb->getDefaultPool()) SimilarToRegex( - *tdbb->getDefaultPool(), 0, + *tdbb->getDefaultPool(), flags, pattern, len, "\\", 1); } @@ -661,7 +661,7 @@ static void explain_pp_bits(const UCHAR bits, Firebird::string& names) } -bool VAL_validate(thread_db* tdbb, USHORT switches) +bool VAL_validate(thread_db* tdbb, USHORT switches, const ValidationDataFilters& filters) { /************************************** * @@ -690,6 +690,8 @@ bool VAL_validate(thread_db* tdbb, USHORT switches) if (!(switches & isc_dpb_no_update)) flags |= Validation::VDR_update; + att->att_validation->setDataFilters(tdbb, filters); + return att->att_validation->run(tdbb, flags); } @@ -877,6 +879,8 @@ Validation::Validation(thread_db* tdbb, UtilSvc* uSvc) vdr_rel_records = NULL; vdr_idx_records = NULL; vdr_page_bitmap = NULL; + vdr_skip_rel_data = false; + vdr_data_skipped = false; vdr_service = uSvc; vdr_lock_tout = -10; @@ -1015,6 +1019,76 @@ void Validation::output(const char* format, ...) } +void Validation::setDataFilters(thread_db* tdbb, const ValidationDataFilters& filters) +{ +/************************************** + * + * s e t D a t a F i l t e r s + * + ************************************** + * + * Functional description + * Compile patterns of relations to validate records of, + * they are case insensitive as in gbak -skip_data / -include_data. + * + **************************************/ + const unsigned flags = SimilarToFlag::CASE_INSENSITIVE; + + vdr_skip_data = createPatternMatcher(tdbb, filters.skipData.nullStr(), flags); + vdr_include_data = createPatternMatcher(tdbb, filters.includeData.nullStr(), flags); + vdr_skip_schema_data = createPatternMatcher(tdbb, filters.skipSchemaData.nullStr(), flags); + vdr_include_schema_data = createPatternMatcher(tdbb, filters.includeSchemaData.nullStr(), flags); +} + + +namespace +{ + enum Pattern { NOT_SET = 0, MATCH = 1, NOT_MATCH = 2 }; + + template + Pattern checkPattern(const AutoPtr& matcher, const Name& name) + { + if (!matcher) + return NOT_SET; + + return matcher->matches(name.c_str(), name.length()) ? MATCH : NOT_MATCH; + } +} + +bool Validation::skipRelationData(const jrd_rel* relation) const +{ +/************************************** + * + * s k i p R e l a t i o n D a t a + * + ************************************** + * + * Functional description + * Check if records of relation should not be validated. + * Uses the same rules as BurpGlobals::skipRelation(). + * + **************************************/ + + // Fine-grained table controlling cases when data must be skipped for a table + static const bool result[3][3] = { + // Include filter + // NS M NM S + { false, false, true}, // NS k + { true, true, true}, // M i + { false, false, true} // NM p + }; + + const QualifiedName& name = relation->getName(); + + const Pattern res1sch = checkPattern(vdr_skip_schema_data, name.schema); + const Pattern res1obj = checkPattern(vdr_skip_data, name.object); + const Pattern res2sch = checkPattern(vdr_include_schema_data, name.schema); + const Pattern res2obj = checkPattern(vdr_include_data, name.object); + + return result[res1sch][res2sch] || result[res1obj][res2obj]; +} + + bool Validation::run(thread_db* tdbb, USHORT flags) { /************************************** @@ -1038,6 +1112,7 @@ bool Validation::run(thread_db* tdbb, USHORT flags) Jrd::ContextPoolHolder context(tdbb, val_pool); vdr_flags = flags; + vdr_data_skipped = false; // initialize validate errors vdr_errors = vdr_warns = vdr_fixed = 0; @@ -1374,10 +1449,12 @@ void Validation::garbage_collect() } } } - else if (!(byte & 1) && (vdr_flags & VDR_records)) + else if (!(byte & 1) && (vdr_flags & VDR_records) && !vdr_data_skipped) { // Page is potentially an orphan - but don't declare it as such - // unless we think we walked all pages + // unless we think we walked all pages. + // Pages of large records and blobs of relations filtered out by + // data filters were not walked. corrupt(VAL_PAG_ORPHAN, 0, number); if (vdr_flags & VDR_update) @@ -1849,7 +1926,7 @@ Validation::RTN Validation::walk_data_page(jrd_rel* relation, ULONG page_number, print_rhd(line->dpg_length, header); } #endif - if (!(header->rhd_flags & rhd_chain) && + if (!(header->rhd_flags & rhd_chain) && !vdr_skip_rel_data && ((header->rhd_flags & rhd_large) || (vdr_flags & VDR_records))) { const RTN result = (header->rhd_flags & rhd_blob) ? @@ -3105,6 +3182,17 @@ Validation::RTN Validation::walk_relation(jrd_rel* relation) vdr_cond_idx.clear(); + // Don't walk records (including large records and blobs) of user relation filtered out + // by -skip_data / -include_data and so on, its pointer, data and index pages are walked + // as usual but without the checks of full validation + const bool skipData = (vdr_flags & VDR_records) && !relation->isSystem() && + skipRelationData(relation); + AutoSetRestoreFlag dataFlags(&vdr_flags, skipData ? VDR_records : 0, false); + AutoSetRestore skipRelData(&vdr_skip_rel_data, skipData); + + if (skipData) + vdr_data_skipped = true; + const bool idxRootOk = (vdr_flags & VDR_records) && !relation->isSystem() ? walk_root(relation, true) == rtn_ok : true; diff --git a/src/jrd/validation.h b/src/jrd/validation.h index b1f42913d2f..6cc554bd3c6 100644 --- a/src/jrd/validation.h +++ b/src/jrd/validation.h @@ -49,6 +49,18 @@ class jrd_rel; class thread_db; +// Regular expressions (SIMILAR TO syntax) passed by gfix -skip_data, -include_data, +// -skip_schema_data and -include_schema_data. Records of relations filtered out by +// them are not validated, but their pages are still walked. + +struct ValidationDataFilters +{ + Firebird::string skipData; + Firebird::string includeData; + Firebird::string skipSchemaData; + Firebird::string includeSchemaData; +}; + // Validation/garbage collection/repair control block class Validation @@ -168,6 +180,12 @@ class Validation Firebird::AutoPtr vdr_tab_excl; Firebird::AutoPtr vdr_idx_incl; Firebird::AutoPtr vdr_idx_excl; + Firebird::AutoPtr vdr_skip_data; + Firebird::AutoPtr vdr_include_data; + Firebird::AutoPtr vdr_skip_schema_data; + Firebird::AutoPtr vdr_include_schema_data; + bool vdr_skip_rel_data; // records of the current relation are not walked due to data filters + bool vdr_data_skipped; // records of some relations were not walked due to data filters int vdr_lock_tout; void checkDPinPP(jrd_rel *relation, ULONG page_number); void checkDPinPIP(jrd_rel *relation, ULONG page_number); @@ -176,6 +194,7 @@ class Validation explicit Validation(thread_db*, Firebird::UtilSvc* uSvc = NULL); ~Validation(); + void setDataFilters(thread_db* tdbb, const ValidationDataFilters& filters); bool run(thread_db* tdbb, USHORT flags); ULONG getInfo(UCHAR item); @@ -206,6 +225,7 @@ class Validation FETCH_CODE fetch_page(bool mark, ULONG, USHORT, WIN*, void*); void release_page(WIN*); void garbage_collect(); + bool skipRelationData(const jrd_rel* relation) const; void parse_args(thread_db*); void output(const char*, ...); diff --git a/src/utilities/fbsvcmgr/fbsvcmgr.cpp b/src/utilities/fbsvcmgr/fbsvcmgr.cpp index 74281d9c1b2..c8d6c42ef97 100644 --- a/src/utilities/fbsvcmgr/fbsvcmgr.cpp +++ b/src/utilities/fbsvcmgr/fbsvcmgr.cpp @@ -510,6 +510,10 @@ constexpr SvcSwitches repairOptions[] = {"rpr_icu", putOption, 0, isc_spb_rpr_icu, 0}, {"rpr_par_workers", putIntArgument, 0, isc_spb_rpr_par_workers, 0}, {"rpr_upgrade_db", putOption, 0, isc_spb_rpr_upgrade_db, 0}, + {"rpr_skip_data", putStringArgument, 0, isc_spb_rpr_skip_data, 0}, + {"rpr_include_data", putStringArgument, 0, isc_spb_rpr_include_data, 0}, + {"rpr_skip_schema_data", putStringArgument, 0, isc_spb_rpr_skip_schema_data, 0}, + {"rpr_include_schema_data", putStringArgument, 0, isc_spb_rpr_include_schema_data, 0}, {0, 0, 0, 0, 0} };