diff --git a/builds/posix/Makefile.in b/builds/posix/Makefile.in index ab416b93ce5..34fa9757ab2 100644 --- a/builds/posix/Makefile.in +++ b/builds/posix/Makefile.in @@ -691,7 +691,7 @@ $(GSPLIT): $(GSPLIT_Objects) $(COMMON_LIB) gfix: $(GFIX) $(GFIX): $(GFIX_Objects) $(COMMON_LIB) - $(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) + $(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) $(RE2LIB) gsec: $(GSEC) diff --git a/builds/win32/msvc15/gfix.vcxproj b/builds/win32/msvc15/gfix.vcxproj index 6aa630b45d0..5bec957bd6b 100644 --- a/builds/win32/msvc15/gfix.vcxproj +++ b/builds/win32/msvc15/gfix.vcxproj @@ -163,7 +163,7 @@ EditAndContinue - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -175,7 +175,7 @@ WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -191,7 +191,7 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -207,7 +207,7 @@ WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console false @@ -224,7 +224,7 @@ WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console @@ -239,7 +239,7 @@ WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions) - comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) + re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies) Console diff --git a/doc/README.gfix_data_filters.md b/doc/README.gfix_data_filters.md new file mode 100644 index 00000000000..2fa9b03ca20 --- /dev/null +++ b/doc/README.gfix_data_filters.md @@ -0,0 +1,70 @@ +# gfix: -SKIP_DATA / -INCLUDE_DATA / -SKIP_SCHEMA_DATA / -INCLUDE_SCHEMA_DATA + +Sometimes a damaged database can't be fully validated or repaired with `gfix -v -full` or `gfix -mend` +because of broken records in some unimportant tables. The switches below allow to exclude records of +such tables from validation, so the rest of the database can still be checked and repaired. + +The switches are similar to the same `gbak` switches and accept a regular expression in SQL syntax +(see `SIMILAR TO`). Matching is case insensitive. + +| Switch | Description | +|---------------------------|--------------------------------------------------------------------| +| `-SKIP_D(ATA)` | skip records validation of tables with matching names | +| `-INCLUDE(_DATA)` | validate records of tables with matching names only | +| `-SKIP_SCHEMA_D(ATA)` | skip records validation of tables in schemas with matching names | +| `-INCLUDE_SCHEMA_D(ATA)` | validate records of tables in schemas with matching names only | + +The rules to decide whether records of a table are validated are the same as in `gbak`: records are skipped +if the table is filtered out either by the table name switches or by the schema name switches. If a table +matches both `-SKIP_DATA` and `-INCLUDE_DATA` (or a schema matches both `-SKIP_SCHEMA_DATA` and +`-INCLUDE_SCHEMA_DATA`), its records are skipped. + +The filters are applied to user tables only, records of system tables are always validated. + +Records of skipped tables are not read at all, including big records (fragmented across several pages) +and blobs, so `-mend` doesn't change them. Pointer pages, data pages and index pages of skipped tables +are still walked and checked as without `-full`: the checks of full validation (record version chains, +index entries against records, consistency of index tree levels) are not performed for them. +Search for orphan pages is not performed when records of any table were skipped, as pages of big +records and blobs of skipped tables were not walked. + +The switches require `-validate` together with `-full` (or `-mend`, which implies both). + +Examples: +```shell +# Validate all records except records of table T1 of any schema +gfix -v -full -skip_data T1 database.fdb + +# Repair the database, skipping records of tables with names starting with LOG_ +gfix -mend -skip_data "LOG\_%" database.fdb + +# Validate records of tables of the schema S1 only +gfix -v -full -include_schema_data S1 database.fdb + +# Validate records of table T1 of the schema S1 only +gfix -v -full -include_schema_data S1 -include_data T1 database.fdb +``` + +## Services API + +The same filters are available in `isc_action_svc_repair` with the following string SPB items: + +| SPB item | fbsvcmgr option | Equivalent | +|-----------------------------------|----------------------------|-----------------------------| +| `isc_spb_rpr_skip_data` | `rpr_skip_data` | `gfix -skip_data` | +| `isc_spb_rpr_include_data` | `rpr_include_data` | `gfix -include_data` | +| `isc_spb_rpr_skip_schema_data` | `rpr_skip_schema_data` | `gfix -skip_schema_data` | +| `isc_spb_rpr_include_schema_data` | `rpr_include_schema_data` | `gfix -include_schema_data` | + +The items require `isc_spb_rpr_validate_db` and `isc_spb_rpr_full` (or `isc_spb_rpr_mend_db`) in `isc_spb_options`, +the order of the items in SPB doesn't matter. + +```shell +fbsvcmgr service_mgr action_repair dbname database.fdb rpr_validate_db rpr_full rpr_skip_data T1 +``` + +## DPB + +`gfix` passes the filters to the engine using the following string DPB items, used together with `isc_dpb_verify`: +`isc_dpb_verify_skip_data`, `isc_dpb_verify_include_data`, `isc_dpb_verify_skip_schema_data`, +`isc_dpb_verify_include_schema_data`. diff --git a/src/alice/alice.cpp b/src/alice/alice.cpp index 06b1ce4357a..a7cd9fa5744 100644 --- a/src/alice/alice.cpp +++ b/src/alice/alice.cpp @@ -50,6 +50,8 @@ #include "../common/utils_proto.h" #include "../common/classes/Switches.h" #include "../common/SimpleStatusVector.h" +#include "../common/SimilarToRegex.h" +#include "../common/isc_f_proto.h" #include "../alice/aliceswi.h" #ifdef HAVE_UNISTD_H @@ -95,6 +97,37 @@ constexpr int ALICE_MSG_FAC = FB_IMPL_MSG_FACILITY_GFIX; static void alice_output(bool error, const SCHAR*, ...) ATTRIBUTE_FORMAT(2,3); +//____________________________________________________________ +// +// Get regular expression argument of -skip_data / -include_data +// and similar switches and check it can be compiled. +// + +static const char* get_data_filter(Firebird::UtilSvc* uSvc, const char**& argv, int& argc, + USHORT missingMsg) +{ + if (--argc <= 0) + ALICE_error(missingMsg); + + const char* const pattern = *argv++; + + try + { + Firebird::string filter(pattern); + if (!uSvc->utf8FileNames()) + ISC_systemToUtf8(filter); + + Firebird::SimilarToRegex matcher(*getDefaultMemoryPool(), + Firebird::SimilarToFlag::CASE_INSENSITIVE, + filter.c_str(), filter.length(), "\\", 1); + } + catch (const Firebird::Exception&) + { + ALICE_error(146, SafeArg() << pattern); // msg 146: invalid regular expression "@1" + } + + return pattern; +} //____________________________________________________________ @@ -143,6 +176,10 @@ int alice(Firebird::UtilSvc* uSvc) tdgbl->ALICE_data.ua_user = NULL; tdgbl->ALICE_data.ua_role = NULL; tdgbl->ALICE_data.ua_password = NULL; + tdgbl->ALICE_data.ua_skip_data = NULL; + tdgbl->ALICE_data.ua_include_data = NULL; + tdgbl->ALICE_data.ua_skip_schema_data = NULL; + tdgbl->ALICE_data.ua_include_schema_data = NULL; #ifdef TRUSTED_AUTH tdgbl->ALICE_data.ua_trusted = false; #endif @@ -477,6 +514,30 @@ int alice(Firebird::UtilSvc* uSvc) else ALICE_error(135); // msg 135: replica mode (none / read_only / read_write) required } + + if (table->in_sw_value & sw_skip_data) + { + tdgbl->ALICE_data.ua_skip_data = get_data_filter(uSvc, argv, argc, 142); + // msg 142: missing regular expression to skip tables + } + + if (table->in_sw_value & sw_include_data) + { + tdgbl->ALICE_data.ua_include_data = get_data_filter(uSvc, argv, argc, 143); + // msg 143: missing regular expression to include tables + } + + if (table->in_sw_value & sw_skip_schema_data) + { + tdgbl->ALICE_data.ua_skip_schema_data = get_data_filter(uSvc, argv, argc, 144); + // msg 144: missing regular expression to skip schemas + } + + if (table->in_sw_value & sw_include_schema_data) + { + tdgbl->ALICE_data.ua_include_schema_data = get_data_filter(uSvc, argv, argc, 145); + // msg 145: missing regular expression to include schemas + } } // put this here since to put it above overly complicates the parsing. @@ -523,6 +584,15 @@ int alice(Firebird::UtilSvc* uSvc) ALICE_exit(FINI_ERROR, tdgbl); } + // Data filters affect records validation only, so they are useless without -full. + // Can't use in_sw_requires since it only looks backwards on command line, + // and services API puts switches on command line in the order of SPB items. + if ((flags & sw_data_filters) && !(flags & sw_full)) + { + ALICE_error(147); + // msg 147: -skip_data, -include_data, -skip_schema_data and -include_schema_data require -full + } + if (!database) { ALICE_error(23); // msg 23: please retry, giving a database name } diff --git a/src/alice/alice.h b/src/alice/alice.h index f2527d6d64d..3bf00b5e7c0 100644 --- a/src/alice/alice.h +++ b/src/alice/alice.h @@ -93,6 +93,10 @@ struct user_action alice_shut_mode ua_shutdown_mode; alice_repl_mode ua_replica_mode; SSHORT ua_parallel_workers; + const char* ua_skip_data; + const char* ua_include_data; + const char* ua_skip_schema_data; + const char* ua_include_schema_data; }; diff --git a/src/alice/aliceswi.h b/src/alice/aliceswi.h index 92b8f373d31..9a7b78e4ace 100644 --- a/src/alice/aliceswi.h +++ b/src/alice/aliceswi.h @@ -67,10 +67,14 @@ inline constexpr SINT64 sw_icu = QUADCONST(0x0000002000000000); inline constexpr SINT64 sw_role = QUADCONST(0x0000004000000000); inline constexpr SINT64 sw_replica = QUADCONST(0x0000008000000000); inline constexpr SINT64 sw_upgrade = QUADCONST(0x0000010000000000); +inline constexpr SINT64 sw_skip_data = QUADCONST(0x0000020000000000); +inline constexpr SINT64 sw_include_data = QUADCONST(0x0000040000000000); +inline constexpr SINT64 sw_skip_schema_data = QUADCONST(0x0000080000000000); +inline constexpr SINT64 sw_include_schema_data = QUADCONST(0x0000100000000000); // Popular combination of compatible switches inline constexpr SINT64 sw_auth_set = sw_user | sw_password | sw_role | sw_fetch_password | sw_trusted_auth; - +inline constexpr SINT64 sw_data_filters = sw_skip_data | sw_include_data | sw_skip_schema_data | sw_include_schema_data; enum alice_switches { @@ -128,7 +132,11 @@ enum alice_switches IN_SW_ALICE_ROLE = 49, IN_SW_ALICE_REPLICA = 50, IN_SW_ALICE_PARALLEL_WORKERS = 51, - IN_SW_ALICE_UPGRADE = 52 + IN_SW_ALICE_UPGRADE = 52, + IN_SW_ALICE_SKIP_DATA = 53, + IN_SW_ALICE_INCLUDE_DATA = 54, + IN_SW_ALICE_SKIP_SCHEMA_DATA = 55, + IN_SW_ALICE_INCLUDE_SCHEMA_DATA = 56 }; static inline constexpr const char* ALICE_SW_ASYNC = "ASYNC"; @@ -190,6 +198,12 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = {IN_SW_ALICE_ICU, isc_spb_rpr_icu, "ICU", sw_icu, 0, sw_shut, false, true, 131, 3, NULL}, // msg 131: \t-icu\t\tfix database to be usable with present ICU version + {IN_SW_ALICE_INCLUDE_DATA, isc_spb_rpr_include_data, "INCLUDE_DATA", sw_include_data, + 0, sw_include_data, false, false, 139, 7, NULL}, + // msg 139: -include(_data) validate records of matching tables only (-full) + {IN_SW_ALICE_INCLUDE_SCHEMA_DATA, isc_spb_rpr_include_schema_data, "INCLUDE_SCHEMA_DATA", sw_include_schema_data, + 0, sw_include_schema_data, false, false, 141, 16, NULL}, + // msg 141: -include_schema_d(ata) validate records of tables in matching schemas only (-full) {IN_SW_ALICE_KILL, isc_spb_rpr_kill_shadows, "KILL_SHADOW", sw_kill, 0, 0, false, true, 36, 1, NULL}, // msg 36: \t-kill\t\tkill all unavailable shadow files @@ -197,7 +211,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = 0, ~(sw_list | sw_auth_set | sw_nolinger), false, true, 37, 1, NULL}, // msg 37: \t-list\t\tshow limbo transactions {IN_SW_ALICE_MEND, isc_spb_rpr_mend_db, "MEND", sw_mend | sw_validate | sw_full, - 0, ~(sw_no_update | sw_auth_set | sw_nolinger), false, true, 38, 2, NULL}, + 0, ~(sw_no_update | sw_auth_set | sw_nolinger | sw_data_filters), false, true, 38, 2, NULL}, // msg 38: \t-mend\t\tprepare corrupt database for backup {IN_SW_ALICE_MODE, 0, "MODE", sw_mode, 0, ~(sw_mode | sw_auth_set | sw_nolinger), false, false, 109, 2, NULL}, @@ -240,6 +254,12 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = {IN_SW_ALICE_SET_DB_SQL_DIALECT, isc_spb_prp_set_sql_dialect, "SQL_DIALECT", sw_set_db_dialect, 0, 0, false, false, 111, 2, NULL}, // msg 111: \t-SQL_dialect\t\set dataabse dialect n + {IN_SW_ALICE_SKIP_DATA, isc_spb_rpr_skip_data, "SKIP_DATA", sw_skip_data, + 0, sw_skip_data, false, false, 138, 6, NULL}, + // msg 138: -skip_d(ata) skip records validation of matching tables (-full) + {IN_SW_ALICE_SKIP_SCHEMA_DATA, isc_spb_rpr_skip_schema_data, "SKIP_SCHEMA_DATA", sw_skip_schema_data, + 0, sw_skip_schema_data, false, false, 140, 13, NULL}, + // msg 140: -skip_schema_d(ata) skip records validation of tables in matching schemas (-full) {IN_SW_ALICE_SWEEP, isc_spb_rpr_sweep_db, "SWEEP", sw_sweep, 0, ~(sw_sweep | sw_auth_set | sw_nolinger), false, true, 45, 2, NULL}, // msg 45: \t-sweep\t\tforce garbage collection @@ -268,7 +288,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = 0, sw_trusted_auth, false, false, 50, 4, NULL}, // msg 50: \t-user\t\tdefault user name {IN_SW_ALICE_VALIDATE, isc_spb_rpr_validate_db, "VALIDATE", sw_validate, - 0, ~(sw_validate | sw_auth_set | sw_nolinger), false, true, 51, 1, NULL}, + 0, ~(sw_validate | sw_auth_set | sw_nolinger | sw_data_filters), false, true, 51, 1, NULL}, // msg 51: \t-validate\tvalidate database structure {IN_SW_ALICE_WRITE, 0, "WRITE", sw_write, 0, ~(sw_write | sw_auth_set | sw_nolinger), false, false, 52, 1, NULL}, @@ -280,6 +300,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] = #endif {IN_SW_ALICE_VERSION, 0, "Z", 0, 0, 0, false, false, 54, 1, NULL}, + // msg 54: \t-z\t\tprint software version number /************************************************************************/ // WARNING: All new switches should be added right before this comments diff --git a/src/alice/exe.cpp b/src/alice/exe.cpp index 4cdfa8da4a3..32a3a16c5df 100644 --- a/src/alice/exe.cpp +++ b/src/alice/exe.cpp @@ -83,7 +83,7 @@ int EXE_action(const TEXT* database, const SINT64 switches) // generate the database parameter block for the attach, // based on the various switches - Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::Tagged, MAX_DPB_SIZE); + Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::dpbList, MAX_DPB_SIZE); buildDpb(dpb, switches); FB_API_HANDLE handle = 0; @@ -159,7 +159,7 @@ int EXE_two_phase(const TEXT* database, const SINT64 switches) // generate the database parameter block for the attach, // based on the various switches - Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::Tagged, MAX_DPB_SIZE); + Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::dpbList, MAX_DPB_SIZE); buildDpb(dpb, switches); FB_API_HANDLE handle = 0; @@ -231,6 +231,20 @@ static void buildDpb(Firebird::ClumpletWriter& dpb, const SINT64 switches) if (switches & sw_ignore) b |= isc_dpb_ignore; dpb.insertByte(isc_dpb_verify, b); + + const user_action& ua = tdgbl->ALICE_data; + + if (switches & sw_skip_data) + dpb.insertString(isc_dpb_verify_skip_data, ua.ua_skip_data); + + if (switches & sw_include_data) + dpb.insertString(isc_dpb_verify_include_data, ua.ua_include_data); + + if (switches & sw_skip_schema_data) + dpb.insertString(isc_dpb_verify_skip_schema_data, ua.ua_skip_schema_data); + + if (switches & sw_include_schema_data) + dpb.insertString(isc_dpb_verify_include_schema_data, ua.ua_include_schema_data); } else if (switches & sw_housekeeping) { dpb.insertInt(isc_dpb_sweep_interval, tdgbl->ALICE_data.ua_sweep_interval); diff --git a/src/common/IntlParametersBlock.cpp b/src/common/IntlParametersBlock.cpp index 7830b40a181..01447dc6da3 100644 --- a/src/common/IntlParametersBlock.cpp +++ b/src/common/IntlParametersBlock.cpp @@ -196,6 +196,10 @@ IntlParametersBlock::TagType IntlDpb::checkTag(UCHAR tag, const char** tagName) FB_IPB_TAG(isc_dpb_os_user); FB_IPB_TAG(isc_dpb_owner); FB_IPB_TAG(isc_dpb_search_path); + FB_IPB_TAG(isc_dpb_verify_skip_data); + FB_IPB_TAG(isc_dpb_verify_include_data); + FB_IPB_TAG(isc_dpb_verify_skip_schema_data); + FB_IPB_TAG(isc_dpb_verify_include_schema_data); return TAG_STRING; default: return TAG_SKIP; @@ -276,6 +280,10 @@ IntlParametersBlock::TagType IntlSpbStart::checkTag(UCHAR tag, const char** tagN switch (tag) { FB_IPB_TAG(isc_spb_tra_db_path); + FB_IPB_TAG(isc_spb_rpr_skip_data); + FB_IPB_TAG(isc_spb_rpr_include_data); + FB_IPB_TAG(isc_spb_rpr_skip_schema_data); + FB_IPB_TAG(isc_spb_rpr_include_schema_data); return TAG_STRING; } break; diff --git a/src/common/classes/ClumpletReader.cpp b/src/common/classes/ClumpletReader.cpp index 32f24050c63..e1a246cd636 100644 --- a/src/common/classes/ClumpletReader.cpp +++ b/src/common/classes/ClumpletReader.cpp @@ -346,6 +346,10 @@ ClumpletReader::ClumpletType ClumpletReader::getClumpletType(UCHAR tag) const switch (tag) { case isc_spb_dbname: + case isc_spb_rpr_skip_data: + case isc_spb_rpr_include_data: + case isc_spb_rpr_skip_schema_data: + case isc_spb_rpr_include_schema_data: return StringSpb; case isc_spb_options: case isc_spb_rpr_commit_trans: diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index 3831a26788a..f7c73fef506 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -138,6 +138,10 @@ #define isc_dpb_search_path 105 #define isc_dpb_blr_request_search_path 106 #define isc_dpb_gbak_restore_has_schema 107 +#define isc_dpb_verify_skip_data 108 +#define isc_dpb_verify_include_data 109 +#define isc_dpb_verify_skip_schema_data 110 +#define isc_dpb_verify_include_schema_data 111 /**************************************************/ @@ -537,6 +541,10 @@ #define isc_spb_rpr_rollback_trans_64 50 #define isc_spb_rpr_recover_two_phase_64 51 #define isc_spb_rpr_par_workers 52 +#define isc_spb_rpr_skip_data 53 +#define isc_spb_rpr_include_data 54 +#define isc_spb_rpr_skip_schema_data 55 +#define isc_spb_rpr_include_schema_data 56 #define isc_spb_rpr_validate_db 0x01 #define isc_spb_rpr_sweep_db 0x02 diff --git a/src/include/firebird/impl/msg/gfix.h b/src/include/firebird/impl/msg/gfix.h index f8f5135e0da..e277a762f5c 100644 --- a/src/include/firebird/impl/msg/gfix.h +++ b/src/include/firebird/impl/msg/gfix.h @@ -135,3 +135,13 @@ FB_IMPL_MSG_SYMBOL(GFIX, 134, gfix_opt_repl, " -repl(ica) replica mo FB_IMPL_MSG_SYMBOL(GFIX, 135, gfix_repl_mode_req, "replica mode (none / read_only / read_write) required") FB_IMPL_MSG_SYMBOL(GFIX, 136, gfix_opt_parallel, " -par(allel) parallel workers (-sweep, -icu)") FB_IMPL_MSG_SYMBOL(GFIX, 137, gfix_opt_upgrade, " -up(grade) upgrade database ODS") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 138, " -skip_d(ata) skip records validation of matching tables (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 139, " -include(_data) validate records of matching tables only (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 140, " -skip_schema_d(ata) skip records validation of tables in matching schemas (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 141, " -include_schema_d(ata) validate records of tables in matching schemas only (-full)") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 142, "missing regular expression to skip tables") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 143, "missing regular expression to include tables") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 144, "missing regular expression to skip schemas") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 145, "missing regular expression to include schemas") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 146, "invalid regular expression \"@1\"") +FB_IMPL_MSG_NO_SYMBOL(GFIX, 147, "-skip_data, -include_data, -skip_schema_data and -include_schema_data require -full") diff --git a/src/include/gen/Firebird.pas b/src/include/gen/Firebird.pas index 393399381a7..d1a16e25912 100644 --- a/src/include/gen/Firebird.pas +++ b/src/include/gen/Firebird.pas @@ -4341,6 +4341,10 @@ IPerformanceStatsImpl = class(IPerformanceStats) isc_dpb_search_path = byte(105); isc_dpb_blr_request_search_path = byte(106); isc_dpb_gbak_restore_has_schema = byte(107); + isc_dpb_verify_skip_data = byte(108); + isc_dpb_verify_include_data = byte(109); + isc_dpb_verify_skip_schema_data = byte(110); + isc_dpb_verify_include_schema_data = byte(111); isc_dpb_address = byte(1); isc_dpb_addr_protocol = byte(1); isc_dpb_addr_endpoint = byte(2); @@ -4570,6 +4574,10 @@ IPerformanceStatsImpl = class(IPerformanceStats) isc_spb_rpr_rollback_trans_64 = byte(50); isc_spb_rpr_recover_two_phase_64 = byte(51); isc_spb_rpr_par_workers = byte(52); + isc_spb_rpr_skip_data = byte(53); + isc_spb_rpr_include_data = byte(54); + isc_spb_rpr_skip_schema_data = byte(55); + isc_spb_rpr_include_schema_data = byte(56); isc_spb_rpr_validate_db = $01; isc_spb_rpr_sweep_db = $02; isc_spb_rpr_mend_db = $04; diff --git a/src/jrd/jrd.cpp b/src/jrd/jrd.cpp index fdb63403818..6efd2a984db 100644 --- a/src/jrd/jrd.cpp +++ b/src/jrd/jrd.cpp @@ -1069,6 +1069,7 @@ namespace Jrd string dpb_decfloat_round; string dpb_decfloat_traps; string dpb_owner; + ValidationDataFilters dpb_verify_filters; Firebird::ObjectsArray dpb_schema_search_path; Firebird::ObjectsArray dpb_blr_request_schema_search_path; @@ -2096,7 +2097,7 @@ JAttachment* JProvider::internalAttach(CheckStatusWrapper* user_status, const ch AutoSetRestoreFlag noCleanup(&attachment->att_flags, ATT_no_cleanup, true); VIO_fini(tdbb); - if (!VAL_validate(tdbb, options.dpb_verify)) + if (!VAL_validate(tdbb, options.dpb_verify, options.dpb_verify_filters)) ERR_punt(); } @@ -7050,6 +7051,22 @@ void DatabaseOptions::get(const UCHAR* dpb, FB_SIZE_T dpb_length, bool& invalid_ dpb_flags |= DBB_damaged; break; + case isc_dpb_verify_skip_data: + getString(rdr, dpb_verify_filters.skipData); + break; + + case isc_dpb_verify_include_data: + getString(rdr, dpb_verify_filters.includeData); + break; + + case isc_dpb_verify_skip_schema_data: + getString(rdr, dpb_verify_filters.skipSchemaData); + break; + + case isc_dpb_verify_include_schema_data: + getString(rdr, dpb_verify_filters.includeSchemaData); + break; + case isc_dpb_trace: rdr.getInt(); break; diff --git a/src/jrd/svc.cpp b/src/jrd/svc.cpp index f08a0b8024f..c12b781d249 100644 --- a/src/jrd/svc.cpp +++ b/src/jrd/svc.cpp @@ -3047,6 +3047,18 @@ bool Service::process_switches(ClumpletReader& spb, string& switches) } get_action_svc_data(spb, switches, bigint); break; + + case isc_spb_rpr_skip_data: + case isc_spb_rpr_include_data: + case isc_spb_rpr_skip_schema_data: + case isc_spb_rpr_include_schema_data: + if (!get_action_svc_parameter(spb.getClumpTag(), alice_in_sw_table, switches)) + { + return false; + } + get_action_svc_string(spb, switches); + break; + case isc_spb_prp_write_mode: case isc_spb_prp_access_mode: case isc_spb_prp_reserve_space: diff --git a/src/jrd/val_proto.h b/src/jrd/val_proto.h index 1272059dd14..dc33300c812 100644 --- a/src/jrd/val_proto.h +++ b/src/jrd/val_proto.h @@ -24,7 +24,12 @@ #ifndef JRD_VAL_PROTO_H #define JRD_VAL_PROTO_H -bool VAL_validate(Jrd::thread_db*, USHORT); +namespace Jrd +{ + struct ValidationDataFilters; +} + +bool VAL_validate(Jrd::thread_db*, USHORT, const Jrd::ValidationDataFilters&); int VAL_service(Firebird::UtilSvc*); inline constexpr int IN_SW_VAL_TAB_INCL = 1; diff --git a/src/jrd/validation.cpp b/src/jrd/validation.cpp index b020cc5b3cc..a3aafa42f30 100644 --- a/src/jrd/validation.cpp +++ b/src/jrd/validation.cpp @@ -592,7 +592,7 @@ static void print_rhd(USHORT, const rhd*); #endif -static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern) +static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern, unsigned flags = 0) { SimilarToRegex* matcher = NULL; try @@ -604,7 +604,7 @@ static SimilarToRegex* createPatternMatcher(thread_db* tdbb, const char* pattern //// TODO: Should this be different than trace and replication //// and use case sensitive matcher? matcher = FB_NEW_POOL(*tdbb->getDefaultPool()) SimilarToRegex( - *tdbb->getDefaultPool(), 0, + *tdbb->getDefaultPool(), flags, pattern, len, "\\", 1); } @@ -661,7 +661,7 @@ static void explain_pp_bits(const UCHAR bits, Firebird::string& names) } -bool VAL_validate(thread_db* tdbb, USHORT switches) +bool VAL_validate(thread_db* tdbb, USHORT switches, const ValidationDataFilters& filters) { /************************************** * @@ -690,6 +690,8 @@ bool VAL_validate(thread_db* tdbb, USHORT switches) if (!(switches & isc_dpb_no_update)) flags |= Validation::VDR_update; + att->att_validation->setDataFilters(tdbb, filters); + return att->att_validation->run(tdbb, flags); } @@ -877,6 +879,8 @@ Validation::Validation(thread_db* tdbb, UtilSvc* uSvc) vdr_rel_records = NULL; vdr_idx_records = NULL; vdr_page_bitmap = NULL; + vdr_skip_rel_data = false; + vdr_data_skipped = false; vdr_service = uSvc; vdr_lock_tout = -10; @@ -1015,6 +1019,76 @@ void Validation::output(const char* format, ...) } +void Validation::setDataFilters(thread_db* tdbb, const ValidationDataFilters& filters) +{ +/************************************** + * + * s e t D a t a F i l t e r s + * + ************************************** + * + * Functional description + * Compile patterns of relations to validate records of, + * they are case insensitive as in gbak -skip_data / -include_data. + * + **************************************/ + const unsigned flags = SimilarToFlag::CASE_INSENSITIVE; + + vdr_skip_data = createPatternMatcher(tdbb, filters.skipData.nullStr(), flags); + vdr_include_data = createPatternMatcher(tdbb, filters.includeData.nullStr(), flags); + vdr_skip_schema_data = createPatternMatcher(tdbb, filters.skipSchemaData.nullStr(), flags); + vdr_include_schema_data = createPatternMatcher(tdbb, filters.includeSchemaData.nullStr(), flags); +} + + +namespace +{ + enum Pattern { NOT_SET = 0, MATCH = 1, NOT_MATCH = 2 }; + + template + Pattern checkPattern(const AutoPtr& matcher, const Name& name) + { + if (!matcher) + return NOT_SET; + + return matcher->matches(name.c_str(), name.length()) ? MATCH : NOT_MATCH; + } +} + +bool Validation::skipRelationData(const jrd_rel* relation) const +{ +/************************************** + * + * s k i p R e l a t i o n D a t a + * + ************************************** + * + * Functional description + * Check if records of relation should not be validated. + * Uses the same rules as BurpGlobals::skipRelation(). + * + **************************************/ + + // Fine-grained table controlling cases when data must be skipped for a table + static const bool result[3][3] = { + // Include filter + // NS M NM S + { false, false, true}, // NS k + { true, true, true}, // M i + { false, false, true} // NM p + }; + + const QualifiedName& name = relation->getName(); + + const Pattern res1sch = checkPattern(vdr_skip_schema_data, name.schema); + const Pattern res1obj = checkPattern(vdr_skip_data, name.object); + const Pattern res2sch = checkPattern(vdr_include_schema_data, name.schema); + const Pattern res2obj = checkPattern(vdr_include_data, name.object); + + return result[res1sch][res2sch] || result[res1obj][res2obj]; +} + + bool Validation::run(thread_db* tdbb, USHORT flags) { /************************************** @@ -1038,6 +1112,7 @@ bool Validation::run(thread_db* tdbb, USHORT flags) Jrd::ContextPoolHolder context(tdbb, val_pool); vdr_flags = flags; + vdr_data_skipped = false; // initialize validate errors vdr_errors = vdr_warns = vdr_fixed = 0; @@ -1374,10 +1449,12 @@ void Validation::garbage_collect() } } } - else if (!(byte & 1) && (vdr_flags & VDR_records)) + else if (!(byte & 1) && (vdr_flags & VDR_records) && !vdr_data_skipped) { // Page is potentially an orphan - but don't declare it as such - // unless we think we walked all pages + // unless we think we walked all pages. + // Pages of large records and blobs of relations filtered out by + // data filters were not walked. corrupt(VAL_PAG_ORPHAN, 0, number); if (vdr_flags & VDR_update) @@ -1849,7 +1926,7 @@ Validation::RTN Validation::walk_data_page(jrd_rel* relation, ULONG page_number, print_rhd(line->dpg_length, header); } #endif - if (!(header->rhd_flags & rhd_chain) && + if (!(header->rhd_flags & rhd_chain) && !vdr_skip_rel_data && ((header->rhd_flags & rhd_large) || (vdr_flags & VDR_records))) { const RTN result = (header->rhd_flags & rhd_blob) ? @@ -3105,6 +3182,17 @@ Validation::RTN Validation::walk_relation(jrd_rel* relation) vdr_cond_idx.clear(); + // Don't walk records (including large records and blobs) of user relation filtered out + // by -skip_data / -include_data and so on, its pointer, data and index pages are walked + // as usual but without the checks of full validation + const bool skipData = (vdr_flags & VDR_records) && !relation->isSystem() && + skipRelationData(relation); + AutoSetRestoreFlag dataFlags(&vdr_flags, skipData ? VDR_records : 0, false); + AutoSetRestore skipRelData(&vdr_skip_rel_data, skipData); + + if (skipData) + vdr_data_skipped = true; + const bool idxRootOk = (vdr_flags & VDR_records) && !relation->isSystem() ? walk_root(relation, true) == rtn_ok : true; diff --git a/src/jrd/validation.h b/src/jrd/validation.h index b1f42913d2f..6cc554bd3c6 100644 --- a/src/jrd/validation.h +++ b/src/jrd/validation.h @@ -49,6 +49,18 @@ class jrd_rel; class thread_db; +// Regular expressions (SIMILAR TO syntax) passed by gfix -skip_data, -include_data, +// -skip_schema_data and -include_schema_data. Records of relations filtered out by +// them are not validated, but their pages are still walked. + +struct ValidationDataFilters +{ + Firebird::string skipData; + Firebird::string includeData; + Firebird::string skipSchemaData; + Firebird::string includeSchemaData; +}; + // Validation/garbage collection/repair control block class Validation @@ -168,6 +180,12 @@ class Validation Firebird::AutoPtr vdr_tab_excl; Firebird::AutoPtr vdr_idx_incl; Firebird::AutoPtr vdr_idx_excl; + Firebird::AutoPtr vdr_skip_data; + Firebird::AutoPtr vdr_include_data; + Firebird::AutoPtr vdr_skip_schema_data; + Firebird::AutoPtr vdr_include_schema_data; + bool vdr_skip_rel_data; // records of the current relation are not walked due to data filters + bool vdr_data_skipped; // records of some relations were not walked due to data filters int vdr_lock_tout; void checkDPinPP(jrd_rel *relation, ULONG page_number); void checkDPinPIP(jrd_rel *relation, ULONG page_number); @@ -176,6 +194,7 @@ class Validation explicit Validation(thread_db*, Firebird::UtilSvc* uSvc = NULL); ~Validation(); + void setDataFilters(thread_db* tdbb, const ValidationDataFilters& filters); bool run(thread_db* tdbb, USHORT flags); ULONG getInfo(UCHAR item); @@ -206,6 +225,7 @@ class Validation FETCH_CODE fetch_page(bool mark, ULONG, USHORT, WIN*, void*); void release_page(WIN*); void garbage_collect(); + bool skipRelationData(const jrd_rel* relation) const; void parse_args(thread_db*); void output(const char*, ...); diff --git a/src/utilities/fbsvcmgr/fbsvcmgr.cpp b/src/utilities/fbsvcmgr/fbsvcmgr.cpp index 74281d9c1b2..c8d6c42ef97 100644 --- a/src/utilities/fbsvcmgr/fbsvcmgr.cpp +++ b/src/utilities/fbsvcmgr/fbsvcmgr.cpp @@ -510,6 +510,10 @@ constexpr SvcSwitches repairOptions[] = {"rpr_icu", putOption, 0, isc_spb_rpr_icu, 0}, {"rpr_par_workers", putIntArgument, 0, isc_spb_rpr_par_workers, 0}, {"rpr_upgrade_db", putOption, 0, isc_spb_rpr_upgrade_db, 0}, + {"rpr_skip_data", putStringArgument, 0, isc_spb_rpr_skip_data, 0}, + {"rpr_include_data", putStringArgument, 0, isc_spb_rpr_include_data, 0}, + {"rpr_skip_schema_data", putStringArgument, 0, isc_spb_rpr_skip_schema_data, 0}, + {"rpr_include_schema_data", putStringArgument, 0, isc_spb_rpr_include_schema_data, 0}, {0, 0, 0, 0, 0} };