Refactor tests which use debug_dtm_action to use faultinjector

.abi-check/7.2.0_arenadata7/postgres.symbols.ignore

@@ -1,2 +1,16 @@
 ConfigureNamesInt_gp
+ConfigureNamesEnum_gp
+ConfigureNamesString_gp
+ConfigureNamesBool_gp
 transformCreateSchemaStmt
+FaultInjector_InjectFaultIfSet_out_of_line
+FaultInjector_InjectFaultIfSet_out_of_line_DTX
+FaultInjector_InjectFaultIfSet_out_of_line_SQL
+InjectFault
+Debug_dtm_action
+Debug_dtm_action_nestinglevel
+Debug_dtm_action_primary
+Debug_dtm_action_protocol
+Debug_dtm_action_segment
+Debug_dtm_action_sql_command_tag
+Debug_dtm_action_target

.abi-check/7.2.0_arenadata7/postgres.types.ignore

@@ -1 +1,3 @@
 struct RunningTransactionsData
+struct FaultInjectorEntry_s
+DDLStatement_e

gpcontrib/gp_inject_fault/README

@@ -3,8 +3,12 @@ Fault Injection Framework
 
 Fault is defined as a point of interest in the source code with an
 associated action to be taken when that point is hit during execution.
-Fault points are defined in the instrumented code using
-SIMPLE_FAULT_INJECTOR() macro.  A fault point is identifed by a name.
+Fault points are defined in the instrumented code. Simplest way is to use
+SIMPLE_FAULT_INJECTOR() macro. There are more complex ways, which allows
+to pass some state variables to fault injection point, and inject fault
+only if certain conditions are met. See section **Fault Injection Macros** below.
+
+A fault point is identified by a name.
 This module provides an interface to inject a predefined fault point
 into a running Greenplum cluster by associating an action with the
 fault point.  Action can be error, panic, sleep, skip, infinite_loop,
@@ -60,7 +64,8 @@ several more parameters.  For example:
           '' /* DDL */, '' /* database name */,
           'my_table' /* table name */,
           1 /* start occurrence */, 10 /* end occurrence */,
-          0 /* */, dbid) from
+          0 /* extra arg */, dbid,
+		  -1 /*gp session id*/, 0 /*nesting level*/) from
    gp_segment_configuration where content=1 and role='p';
 
 The above command sets heap_insert fault such that the inserting
@@ -70,6 +75,11 @@ fault point, only if the relation being inserted to has the name
 stop triggering.  The 11th transaction to insert into my_table will
 continue the insert as if no fault was injected.
 
+Interpretation of the extraArg argument depends on fault type. 
+
+There are few versions of this function with lesser number of
+parameters. See gp_inject_fault--1.0.sql.
+
 Fault actions
 -------------
 
@@ -86,7 +96,8 @@ panic
    elog(PANIC)
 
 sleep
-   sleep for specified amount of time
+   sleep for amount of time, specified as extraArg parameter, in
+   seconds.
 
 infinite_loop
    loop until query cancel or terminate signal is received
@@ -107,11 +118,17 @@ skip
    }
 
 reset
-   remove a previously injected fault
+   remove a previously injected fault. It is neccessary to reset fault
+   before injection of other fault with same name and different
+   parameters.
 
 segv
    crash the backend process due to SIGSEGV
 
+exit_no_callbacks
+   terminate backend using C _exit() function with return code from
+   extraArg argument.
+
 interrupt
    simulate cancel interrupt arrival, such that the next
    interrupt processing cycle will cancel the query
@@ -133,6 +150,88 @@ status
       longer be taken because the fault point has been reached maximum
       number of times during execution.
 
+Fault injection parameters
+--------------------------
+
+Following parameters can be used for fault injection function:
+
+*  faultname text - mandatory. Name of fault injection point as defined
+   in the code.
+*  type text - mandatory. See previous section
+*  ddl text - SQL operator name or DTX protocol command which should be
+   processed otherwise fault wouldn't be injected. Pass empty string, if
+   you don't want limit injection by particular operator. See
+   `faultinjector_lists.h` for list of allowed operators. This
+   parameter makes sense only if injection point in the code was defined
+   by macro which accepts `ddlStatement` parameter and some meaningful
+   value is passed to it.
+*  database text - database name which should be processed. Makes sense
+   only if you know that database name was passed to injection point in
+   the code
+*  tablename text - table name which should be processed. Same
+   conditions as in previous case stands.
+*  start_occurrence int4 - number of times injection point is hit when
+   first fault is injected.
+*  end_occurrence int4 - number of times injection point is hit when
+   last fault is injected. Use -1 if you don't want to limit number of
+   hits.
+*  extra_arg int4 - use of this argument depends on fault type. For type
+   'sleep' it is number of seconds to sleep,
+*  db_id int4 - db id (from dbid column of gp_segment_configuration table)
+   mandatory. See above for examples of SQL queries.
+*  gp_session_id int4 - value of global variable gp_session_id (see 
+   src/include/cdb/cdbvars.h) If you don't want to limit faults
+   to particular session, use -1 (value of C constant InvalidGpSessionId).
+*  nestinglevel int4 level of transaction nesting. Makes sense only if
+   its value is passed to the injection point in the code. Use 0 if you
+   don't want limit level, or just use SQL function without this
+   parameter.
+
+Fault Injection Macros
+----------------------
+
+In the C code fault injection point is marked by adding one of macros,
+defined in the faultinjector.h. All macros end up with call of single
+function FaultInjector_InjectFaultIfSet_out_of_line, but this function
+is always wrapped with some macro to avoid function call overhead if no
+fault was injected into particular segment.
+
+```
+FaultInjector_InjectFaultIfSet(const char *faultName,
+    DDLStatement_e  ddlstatement,
+    const char* databaseName,
+	const char* tableName)
+```
+
+Allows to pass database name and tablename. ddlStatement should be
+passed as element of enumeration DDLStatement_e
+
+```
+FaultInjector_InjectFaultIfSet_SQL(
+     const char faultName,
+	 const char SQLStatement,
+	 int nestingLevel)
+```
+
+Allows to specify SQL statement as commandTag string which can be
+derived from parsetree node via CreateCommandTag() function (see
+src/include/tcop/utility.h). For MPPEXEC statements command tags are
+created by explicit code at the beginning of exec_mpp_query() function.
+
+```
+FaultInjector_InjectFaultIfSet_DTX(
+      const char * faultName,
+	  DtxProtocolCommand dtxProtocol,
+	  int nestingLevel)
+```
+Allows to pass DTX protocol command as value of enumeration
+DtxProtocolCommand and subtransaction nesting level.
+
+```
+SIMPLE_FAULT_INJECTOR(const char *faultName)
+```
+Already discussed in the introduction section.
+
 NOTES
 -----
 
@@ -144,5 +243,5 @@ many backends, such as in parallel regress tests.
 associates actions with them.
 
 * We let some background process ignore all but a few faults. If one wants
-to test fault injection in background processese, add the exception in
+to test fault injection in background processes, add the exception in
 checkBgProcessSkipFault().

gpcontrib/gp_inject_fault/gp_inject_fault--1.0.sql

@@ -14,11 +14,28 @@ CREATE FUNCTION @extschema@.gp_inject_fault(
   end_occurrence int4,
   extra_arg int4,
   db_id int4,
-  gp_session_id int4)
+  gp_session_id int4,
+  nestinglevel int4)
 RETURNS text
 AS 'MODULE_PATHNAME'
 LANGUAGE C VOLATILE STRICT NO SQL;
 
+-- simpler version without nesting level
+
+CREATE FUNCTION @extschema@.gp_inject_fault(
+  faultname text,
+  type text,
+  ddl text,
+  database text,
+  tablename text,
+  start_occurrence int4,
+  end_occurrence int4,
+  extra_arg int4,
+  db_id int4,
+  gp_session_id int4)
+RETURNS text
+AS $$ select @extschema@.gp_inject_fault($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, 0) $$
+LANGUAGE SQL;
 -- Simpler version, without specific session id.
 CREATE FUNCTION @extschema@.gp_inject_fault(
   faultname text,
@@ -31,7 +48,7 @@ CREATE FUNCTION @extschema@.gp_inject_fault(
   extra_arg int4,
   db_id int4)
 RETURNS text
-AS $$ select @extschema@.gp_inject_fault($1, $2, $3, $4, $5, $6, $7, $8, $9, -1) $$
+AS $$ select @extschema@.gp_inject_fault($1, $2, $3, $4, $5, $6, $7, $8, $9, -1, 0) $$
 LANGUAGE SQL;
 
 -- Simpler version, trigger only one time, occurrence start at 1 and
@@ -41,7 +58,7 @@ CREATE FUNCTION @extschema@.gp_inject_fault(
   type text,
   db_id int4)
 RETURNS text
-AS $$ select @extschema@.gp_inject_fault($1, $2, '', '', '', 1, 1, 0, $3, -1) $$
+AS $$ select @extschema@.gp_inject_fault($1, $2, '', '', '', 1, 1, 0, $3, -1, 0) $$
 LANGUAGE SQL;
 
 -- Simpler version, trigger only one time, occurrence start at 1 and
@@ -52,7 +69,7 @@ CREATE FUNCTION @extschema@.gp_inject_fault(
   db_id int4,
   gp_session_id int4)
 RETURNS text
-AS $$ select @extschema@.gp_inject_fault($1, $2, '', '', '', 1, 1, 0, $3, $4) $$
+AS $$ select @extschema@.gp_inject_fault($1, $2, '', '', '', 1, 1, 0, $3, $4, 0) $$
 LANGUAGE SQL;
 
 -- Simpler version, always trigger until fault is reset.
@@ -61,7 +78,7 @@ CREATE FUNCTION @extschema@.gp_inject_fault_infinite(
   type text,
   db_id int4)
 RETURNS text
-AS $$ select @extschema@.gp_inject_fault($1, $2, '', '', '', 1, -1, 0, $3, -1) $$
+AS $$ select @extschema@.gp_inject_fault($1, $2, '', '', '', 1, -1, 0, $3, -1, 0) $$
 LANGUAGE SQL;
 
 -- Simpler version to avoid confusion for wait_until_triggered fault.
@@ -72,7 +89,7 @@ CREATE FUNCTION @extschema@.gp_wait_until_triggered_fault(
   numtimestriggered int4,
   db_id int4)
 RETURNS text
-AS $$ select @extschema@.gp_inject_fault($1, 'wait_until_triggered', '', '', '', 1, 1, $2, $3, -1) $$
+AS $$ select @extschema@.gp_inject_fault($1, 'wait_until_triggered', '', '', '', 1, 1, $2, $3, -1, 0) $$
 LANGUAGE SQL;
 
 CREATE OR REPLACE FUNCTION @extschema@.insert_noop_xlog_record()

gpcontrib/gp_inject_fault/gp_inject_fault.c

@@ -102,6 +102,7 @@ gp_inject_fault(PG_FUNCTION_ARGS)
 	 * others mean the fault could only be triggered by the specific session.
 	 */
 	int		gpSessionid = PG_GETARG_INT32(9);
+	int		nestingLevel = PG_GETARG_INT32(10);
 	char	*hostname;
 	int		port;
 	char	*response;
@@ -110,8 +111,9 @@ gp_inject_fault(PG_FUNCTION_ARGS)
 	if (GpIdentity.dbid == dbid)
 	{
 		response = InjectFault(
-			faultName, type, ddlStatement, databaseName,
-			tableName, startOccurrence, endOccurrence, extraArg, gpSessionid);
+			faultName, type, ddlStatement, databaseName, tableName,
+			startOccurrence, endOccurrence, extraArg, gpSessionid,
+			nestingLevel);
 		if (!response)
 			elog(ERROR, "failed to inject fault locally (dbid %d)", dbid);
 		if (strncmp(response, "Success:",  strlen("Success:")) != 0)
@@ -142,16 +144,17 @@ gp_inject_fault(PG_FUNCTION_ARGS)
 			databaseName = "#";
 		if (!tableName || tableName[0] == '\0')
 			tableName = "#";
-		snprintf(msg, 1024, "faultname=%s type=%s ddl=%s db=%s table=%s "
-				 "start=%d end=%d extra=%d sid=%d ",
+		snprintf(msg, 1024, "faultname=%s type=%s ddl='%s' db=%s table=%s "
+				 "start=%d end=%d extra=%d sid=%d nestinglevel=%d",
 				 faultName, type,
 				 ddlStatement,
 				 databaseName,
 				 tableName,
 				 startOccurrence,
 				 endOccurrence,
 				 extraArg,
-				 gpSessionid);
+				 gpSessionid,
+				 nestingLevel);
 		res = PQexec(conn, msg);
 		if (PQresultStatus(res) != PGRES_TUPLES_OK)
 			elog(ERROR, "failed to inject fault: %s", PQerrorMessage(conn));