Add data filters to gfix validation: -skip_data, -include_data, -skip_schema_data, -include_schema_data

builds/posix/Makefile.in

@@ -691,7 +691,7 @@ $(GSPLIT):		$(GSPLIT_Objects) $(COMMON_LIB)
 gfix:			$(GFIX)
 
 $(GFIX):		$(GFIX_Objects) $(COMMON_LIB)
-	$(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS)
+	$(EXE_LINK) $(EXE_LINK_OPTIONS) $^ -o $@ $(FIREBIRD_LIBRARY_LINK) $(LINK_LIBS) $(RE2LIB)
 
 gsec:			$(GSEC)
 

builds/win32/msvc15/gfix.vcxproj

@@ -163,7 +163,7 @@
       <DebugInformationFormat>EditAndContinue</DebugInformationFormat>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem>Console</SubSystem>
       <RandomizedBaseAddress>false</RandomizedBaseAddress>
       <DataExecutionPrevention>
@@ -175,7 +175,7 @@
       <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem>Console</SubSystem>
       <RandomizedBaseAddress>false</RandomizedBaseAddress>
       <DataExecutionPrevention>
@@ -191,7 +191,7 @@
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem>Console</SubSystem>
       <RandomizedBaseAddress>false</RandomizedBaseAddress>
       <DataExecutionPrevention>
@@ -207,7 +207,7 @@
       <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem>Console</SubSystem>
       <RandomizedBaseAddress>false</RandomizedBaseAddress>
       <DataExecutionPrevention>
@@ -224,7 +224,7 @@
       <PreprocessorDefinitions>WIN32;_DEBUG;_CONSOLE;DEV_BUILD;SUPERCLIENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem>Console</SubSystem>
       <DataExecutionPrevention>
       </DataExecutionPrevention>
@@ -239,7 +239,7 @@
       <PreprocessorDefinitions>WIN32;NDEBUG;_CONSOLE;SUPERCLIENT;%(PreprocessorDefinitions)</PreprocessorDefinitions>
     </ClCompile>
     <Link>
-      <AdditionalDependencies>comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>re2.lib;comctl32.lib;ws2_32.lib;mpr.lib;version.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <SubSystem>Console</SubSystem>
       <DataExecutionPrevention>
       </DataExecutionPrevention>

doc/README.gfix_data_filters.md

@@ -0,0 +1,70 @@
+# gfix: -SKIP_DATA / -INCLUDE_DATA / -SKIP_SCHEMA_DATA / -INCLUDE_SCHEMA_DATA
+
+Sometimes a damaged database can't be fully validated or repaired with `gfix -v -full` or `gfix -mend`
+because of broken records in some unimportant tables. The switches below allow to exclude records of
+such tables from validation, so the rest of the database can still be checked and repaired.
+
+The switches are similar to the same `gbak` switches and accept a regular expression in SQL syntax
+(see `SIMILAR TO`). Matching is case insensitive.
+
+| Switch                    | Description                                                        |
+|---------------------------|--------------------------------------------------------------------|
+| `-SKIP_D(ATA)`            | skip records validation of tables with matching names              |
+| `-INCLUDE(_DATA)`         | validate records of tables with matching names only                |
+| `-SKIP_SCHEMA_D(ATA)`     | skip records validation of tables in schemas with matching names   |
+| `-INCLUDE_SCHEMA_D(ATA)`  | validate records of tables in schemas with matching names only     |
+
+The rules to decide whether records of a table are validated are the same as in `gbak`: records are skipped
+if the table is filtered out either by the table name switches or by the schema name switches. If a table
+matches both `-SKIP_DATA` and `-INCLUDE_DATA` (or a schema matches both `-SKIP_SCHEMA_DATA` and
+`-INCLUDE_SCHEMA_DATA`), its records are skipped.
+
+The filters are applied to user tables only, records of system tables are always validated.
+
+Records of skipped tables are not read at all, including big records (fragmented across several pages)
+and blobs, so `-mend` doesn't change them. Pointer pages, data pages and index pages of skipped tables
+are still walked and checked as without `-full`: the checks of full validation (record version chains,
+index entries against records, consistency of index tree levels) are not performed for them.
+Search for orphan pages is not performed when records of any table were skipped, as pages of big
+records and blobs of skipped tables were not walked.
+
+The switches require `-validate` together with `-full` (or `-mend`, which implies both).
+
+Examples:
+```shell
+# Validate all records except records of table T1 of any schema
+gfix -v -full -skip_data T1 database.fdb
+
+# Repair the database, skipping records of tables with names starting with LOG_
+gfix -mend -skip_data "LOG\_%" database.fdb
+
+# Validate records of tables of the schema S1 only
+gfix -v -full -include_schema_data S1 database.fdb
+
+# Validate records of table T1 of the schema S1 only
+gfix -v -full -include_schema_data S1 -include_data T1 database.fdb
+```
+
+## Services API
+
+The same filters are available in `isc_action_svc_repair` with the following string SPB items:
+
+| SPB item                          | fbsvcmgr option            | Equivalent                  |
+|-----------------------------------|----------------------------|-----------------------------|
+| `isc_spb_rpr_skip_data`           | `rpr_skip_data`            | `gfix -skip_data`           |
+| `isc_spb_rpr_include_data`        | `rpr_include_data`         | `gfix -include_data`        |
+| `isc_spb_rpr_skip_schema_data`    | `rpr_skip_schema_data`     | `gfix -skip_schema_data`    |
+| `isc_spb_rpr_include_schema_data` | `rpr_include_schema_data`  | `gfix -include_schema_data` |
+
+The items require `isc_spb_rpr_validate_db` and `isc_spb_rpr_full` (or `isc_spb_rpr_mend_db`) in `isc_spb_options`,
+the order of the items in SPB doesn't matter.
+
+```shell
+fbsvcmgr service_mgr action_repair dbname database.fdb rpr_validate_db rpr_full rpr_skip_data T1
+```
+
+## DPB
+
+`gfix` passes the filters to the engine using the following string DPB items, used together with `isc_dpb_verify`:
+`isc_dpb_verify_skip_data`, `isc_dpb_verify_include_data`, `isc_dpb_verify_skip_schema_data`,
+`isc_dpb_verify_include_schema_data`.

src/alice/alice.cpp

@@ -50,6 +50,8 @@
 #include "../common/utils_proto.h"
 #include "../common/classes/Switches.h"
 #include "../common/SimpleStatusVector.h"
+#include "../common/SimilarToRegex.h"
+#include "../common/isc_f_proto.h"
 #include "../alice/aliceswi.h"
 
 #ifdef HAVE_UNISTD_H
@@ -95,6 +97,37 @@ constexpr int ALICE_MSG_FAC = FB_IMPL_MSG_FACILITY_GFIX;
 
 static void alice_output(bool error, const SCHAR*, ...) ATTRIBUTE_FORMAT(2,3);
 
+//____________________________________________________________
+//
+//	Get regular expression argument of -skip_data / -include_data
+//	and similar switches and check it can be compiled.
+//
+
+static const char* get_data_filter(Firebird::UtilSvc* uSvc, const char**& argv, int& argc,
+	USHORT missingMsg)
+{
+	if (--argc <= 0)
+		ALICE_error(missingMsg);
+
+	const char* const pattern = *argv++;
+
+	try
+	{
+		Firebird::string filter(pattern);
+		if (!uSvc->utf8FileNames())
+			ISC_systemToUtf8(filter);
+
+		Firebird::SimilarToRegex matcher(*getDefaultMemoryPool(),
+			Firebird::SimilarToFlag::CASE_INSENSITIVE,
+			filter.c_str(), filter.length(), "\\", 1);
+	}
+	catch (const Firebird::Exception&)
+	{
+		ALICE_error(146, SafeArg() << pattern);	// msg 146: invalid regular expression "@1"
+	}
+
+	return pattern;
+}
 
 
 //____________________________________________________________
@@ -143,6 +176,10 @@ int alice(Firebird::UtilSvc* uSvc)
 	tdgbl->ALICE_data.ua_user = NULL;
 	tdgbl->ALICE_data.ua_role = NULL;
 	tdgbl->ALICE_data.ua_password = NULL;
+	tdgbl->ALICE_data.ua_skip_data = NULL;
+	tdgbl->ALICE_data.ua_include_data = NULL;
+	tdgbl->ALICE_data.ua_skip_schema_data = NULL;
+	tdgbl->ALICE_data.ua_include_schema_data = NULL;
 #ifdef TRUSTED_AUTH
 	tdgbl->ALICE_data.ua_trusted = false;
 #endif
@@ -477,6 +514,30 @@ int alice(Firebird::UtilSvc* uSvc)
 			else
 				ALICE_error(135);	// msg 135: replica mode (none / read_only / read_write) required
 		}
+
+		if (table->in_sw_value & sw_skip_data)
+		{
+			tdgbl->ALICE_data.ua_skip_data = get_data_filter(uSvc, argv, argc, 142);
+			// msg 142: missing regular expression to skip tables
+		}
+
+		if (table->in_sw_value & sw_include_data)
+		{
+			tdgbl->ALICE_data.ua_include_data = get_data_filter(uSvc, argv, argc, 143);
+			// msg 143: missing regular expression to include tables
+		}
+
+		if (table->in_sw_value & sw_skip_schema_data)
+		{
+			tdgbl->ALICE_data.ua_skip_schema_data = get_data_filter(uSvc, argv, argc, 144);
+			// msg 144: missing regular expression to skip schemas
+		}
+
+		if (table->in_sw_value & sw_include_schema_data)
+		{
+			tdgbl->ALICE_data.ua_include_schema_data = get_data_filter(uSvc, argv, argc, 145);
+			// msg 145: missing regular expression to include schemas
+		}
 	}
 
 	// put this here since to put it above overly complicates the parsing.
@@ -523,6 +584,15 @@ int alice(Firebird::UtilSvc* uSvc)
 		ALICE_exit(FINI_ERROR, tdgbl);
 	}
 
+	// Data filters affect records validation only, so they are useless without -full.
+	// Can't use in_sw_requires since it only looks backwards on command line,
+	// and services API puts switches on command line in the order of SPB items.
+	if ((flags & sw_data_filters) && !(flags & sw_full))
+	{
+		ALICE_error(147);
+		// msg 147: -skip_data, -include_data, -skip_schema_data and -include_schema_data require -full
+	}
+
 	if (!database) {
 		ALICE_error(23);	// msg 23: please retry, giving a database name
 	}

src/alice/alice.h

@@ -93,6 +93,10 @@ struct user_action
 	alice_shut_mode ua_shutdown_mode;
 	alice_repl_mode ua_replica_mode;
 	SSHORT ua_parallel_workers;
+	const char* ua_skip_data;
+	const char* ua_include_data;
+	const char* ua_skip_schema_data;
+	const char* ua_include_schema_data;
 };
 
 

src/alice/aliceswi.h

@@ -67,10 +67,14 @@ inline constexpr SINT64 sw_icu				= QUADCONST(0x0000002000000000);
 inline constexpr SINT64 sw_role				= QUADCONST(0x0000004000000000);
 inline constexpr SINT64 sw_replica			= QUADCONST(0x0000008000000000);
 inline constexpr SINT64 sw_upgrade			= QUADCONST(0x0000010000000000);
+inline constexpr SINT64 sw_skip_data		= QUADCONST(0x0000020000000000);
+inline constexpr SINT64 sw_include_data		= QUADCONST(0x0000040000000000);
+inline constexpr SINT64 sw_skip_schema_data	= QUADCONST(0x0000080000000000);
+inline constexpr SINT64 sw_include_schema_data	= QUADCONST(0x0000100000000000);
 
 // Popular combination of compatible switches
 inline constexpr SINT64 sw_auth_set = sw_user | sw_password | sw_role | sw_fetch_password | sw_trusted_auth;
-
+inline constexpr SINT64 sw_data_filters = sw_skip_data | sw_include_data | sw_skip_schema_data | sw_include_schema_data;
 
 enum alice_switches
 {
@@ -128,7 +132,11 @@ enum alice_switches
 	IN_SW_ALICE_ROLE				=	49,
 	IN_SW_ALICE_REPLICA				=	50,
 	IN_SW_ALICE_PARALLEL_WORKERS	=	51,
-	IN_SW_ALICE_UPGRADE				=	52
+	IN_SW_ALICE_UPGRADE				=	52,
+	IN_SW_ALICE_SKIP_DATA			=	53,
+	IN_SW_ALICE_INCLUDE_DATA		=	54,
+	IN_SW_ALICE_SKIP_SCHEMA_DATA	=	55,
+	IN_SW_ALICE_INCLUDE_SCHEMA_DATA	=	56
 };
 
 static inline constexpr const char* ALICE_SW_ASYNC	= "ASYNC";
@@ -190,14 +198,20 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] =
 	{IN_SW_ALICE_ICU, isc_spb_rpr_icu, "ICU", sw_icu,
 		0, sw_shut, false, true, 131, 3, NULL},
 	// msg 131: \t-icu\t\tfix database to be usable with present ICU version
+	{IN_SW_ALICE_INCLUDE_DATA, isc_spb_rpr_include_data, "INCLUDE_DATA", sw_include_data,
+		0, sw_include_data, false, false, 139, 7, NULL},
+	// msg 139: -include(_data) validate records of matching tables only (-full)
+	{IN_SW_ALICE_INCLUDE_SCHEMA_DATA, isc_spb_rpr_include_schema_data, "INCLUDE_SCHEMA_DATA", sw_include_schema_data,
+		0, sw_include_schema_data, false, false, 141, 16, NULL},
+	// msg 141: -include_schema_d(ata) validate records of tables in matching schemas only (-full)
 	{IN_SW_ALICE_KILL, isc_spb_rpr_kill_shadows, "KILL_SHADOW", sw_kill,
 		0, 0, false, true, 36, 1, NULL},
 	// msg 36: \t-kill\t\tkill all unavailable shadow files
 	{IN_SW_ALICE_LIST, isc_spb_rpr_list_limbo_trans, "LIST", sw_list,
 		0, ~(sw_list | sw_auth_set | sw_nolinger), false, true, 37, 1, NULL},
 	// msg 37: \t-list\t\tshow limbo transactions
 	{IN_SW_ALICE_MEND, isc_spb_rpr_mend_db, "MEND", sw_mend | sw_validate | sw_full,
-		0, ~(sw_no_update | sw_auth_set | sw_nolinger), false, true, 38, 2, NULL},
+		0, ~(sw_no_update | sw_auth_set | sw_nolinger | sw_data_filters), false, true, 38, 2, NULL},
 	// msg 38: \t-mend\t\tprepare corrupt database for backup
 	{IN_SW_ALICE_MODE, 0, "MODE", sw_mode,
 		0, ~(sw_mode | sw_auth_set | sw_nolinger), false, false, 109, 2, NULL},
@@ -240,6 +254,12 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] =
 	{IN_SW_ALICE_SET_DB_SQL_DIALECT, isc_spb_prp_set_sql_dialect, "SQL_DIALECT", sw_set_db_dialect,
 		0, 0, false, false, 111, 2, NULL},
 	// msg 111: \t-SQL_dialect\t\set dataabse dialect n
+	{IN_SW_ALICE_SKIP_DATA, isc_spb_rpr_skip_data, "SKIP_DATA", sw_skip_data,
+		0, sw_skip_data, false, false, 138, 6, NULL},
+	// msg 138: -skip_d(ata) skip records validation of matching tables (-full)
+	{IN_SW_ALICE_SKIP_SCHEMA_DATA, isc_spb_rpr_skip_schema_data, "SKIP_SCHEMA_DATA", sw_skip_schema_data,
+		0, sw_skip_schema_data, false, false, 140, 13, NULL},
+	// msg 140: -skip_schema_d(ata) skip records validation of tables in matching schemas (-full)
 	{IN_SW_ALICE_SWEEP, isc_spb_rpr_sweep_db, "SWEEP", sw_sweep,
 		0, ~(sw_sweep | sw_auth_set | sw_nolinger), false, true, 45, 2, NULL},
 	// msg 45: \t-sweep\t\tforce garbage collection
@@ -268,7 +288,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] =
 		0, sw_trusted_auth, false, false, 50, 4, NULL},
 	// msg 50: \t-user\t\tdefault user name
 	{IN_SW_ALICE_VALIDATE, isc_spb_rpr_validate_db, "VALIDATE", sw_validate,
-		0, ~(sw_validate | sw_auth_set | sw_nolinger), false, true, 51, 1, NULL},
+		0, ~(sw_validate | sw_auth_set | sw_nolinger | sw_data_filters), false, true, 51, 1, NULL},
 	// msg 51: \t-validate\tvalidate database structure
 	{IN_SW_ALICE_WRITE, 0, "WRITE", sw_write,
 		0, ~(sw_write | sw_auth_set | sw_nolinger), false, false, 52, 1, NULL},
@@ -280,6 +300,7 @@ static inline constexpr Switches::in_sw_tab_t alice_in_sw_table[] =
 #endif
 	{IN_SW_ALICE_VERSION, 0, "Z", 0,
 		0, 0, false, false, 54, 1, NULL},
+
 	// msg 54: \t-z\t\tprint software version number
 /************************************************************************/
 // WARNING: All new switches should be added right before this comments

src/alice/exe.cpp

@@ -83,7 +83,7 @@ int EXE_action(const TEXT* database, const SINT64 switches)
 
 		// generate the database parameter block for the attach,
 		// based on the various switches
-		Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::Tagged, MAX_DPB_SIZE);
+		Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::dpbList, MAX_DPB_SIZE);
 		buildDpb(dpb, switches);
 
 		FB_API_HANDLE handle = 0;
@@ -159,7 +159,7 @@ int EXE_two_phase(const TEXT* database, const SINT64 switches)
 
 		// generate the database parameter block for the attach,
 		// based on the various switches
-		Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::Tagged, MAX_DPB_SIZE);
+		Firebird::ClumpletWriter dpb(Firebird::ClumpletReader::dpbList, MAX_DPB_SIZE);
 		buildDpb(dpb, switches);
 
 		FB_API_HANDLE handle = 0;
@@ -231,6 +231,20 @@ static void buildDpb(Firebird::ClumpletWriter& dpb, const SINT64 switches)
 		if (switches & sw_ignore)
 			b |= isc_dpb_ignore;
 		dpb.insertByte(isc_dpb_verify, b);
+
+		const user_action& ua = tdgbl->ALICE_data;
+
+		if (switches & sw_skip_data)
+			dpb.insertString(isc_dpb_verify_skip_data, ua.ua_skip_data);
+
+		if (switches & sw_include_data)
+			dpb.insertString(isc_dpb_verify_include_data, ua.ua_include_data);
+
+		if (switches & sw_skip_schema_data)
+			dpb.insertString(isc_dpb_verify_skip_schema_data, ua.ua_skip_schema_data);
+
+		if (switches & sw_include_schema_data)
+			dpb.insertString(isc_dpb_verify_include_schema_data, ua.ua_include_schema_data);
 	}
 	else if (switches & sw_housekeeping) {
 		dpb.insertInt(isc_dpb_sweep_interval, tdgbl->ALICE_data.ua_sweep_interval);

src/common/IntlParametersBlock.cpp

@@ -196,6 +196,10 @@ IntlParametersBlock::TagType IntlDpb::checkTag(UCHAR tag, const char** tagName)
 	FB_IPB_TAG(isc_dpb_os_user);
 	FB_IPB_TAG(isc_dpb_owner);
 	FB_IPB_TAG(isc_dpb_search_path);
+	FB_IPB_TAG(isc_dpb_verify_skip_data);
+	FB_IPB_TAG(isc_dpb_verify_include_data);
+	FB_IPB_TAG(isc_dpb_verify_skip_schema_data);
+	FB_IPB_TAG(isc_dpb_verify_include_schema_data);
 		return TAG_STRING;
 	default:
 		return TAG_SKIP;
@@ -276,6 +280,10 @@ IntlParametersBlock::TagType IntlSpbStart::checkTag(UCHAR tag, const char** tagN
 		switch (tag)
 		{
 		FB_IPB_TAG(isc_spb_tra_db_path);
+		FB_IPB_TAG(isc_spb_rpr_skip_data);
+		FB_IPB_TAG(isc_spb_rpr_include_data);
+		FB_IPB_TAG(isc_spb_rpr_skip_schema_data);
+		FB_IPB_TAG(isc_spb_rpr_include_schema_data);
 			return TAG_STRING;
 		}
 		break;