zinject: separate match counts for delay injections

It turns out we can count a matching delay injection handler earlier, by
counting it once we decide its worth considering if it has a spare lane
for this IO.

With that in hand, it becomes useful to show separate match and inject
counts, as well as the frequency, so update the output and the test to
show all that.

Sponsored-by: Klara, Inc.
Sponsored-by: Wasabi Technology, Inc.
Suggested-by: Alexander Motin <[email protected]>
Signed-off-by: Rob Norris <[email protected]>

Author: robn

cmd/zinject/zinject.c

@@ -484,19 +484,25 @@ print_delay_handler(int id, const char *pool, zinject_record_t *record,
 		return (0);
 
 	if (*count == 0) {
-		(void) printf("%3s  %-15s  %-15s  %-15s  %-16s  %-6s\n",
-		    "ID", "POOL", "DELAY (ms)", "LANES", "GUID", "INJECT");
-		(void) printf("---  ---------------  ---------------  "
-		    "---------------  ----------------  ------\n");
+		(void) printf("%3s  %-15s  %-16s  %-10s  %-5s  %-9s  "
+		    "%-6s  %-6s\n",
+		    "ID", "POOL", "GUID", "DELAY (ms)", "LANES", "FREQ",
+		    "MATCH", "INJECT");
+		(void) printf("---  ---------------  ----------------  "
+		    "----------  -----  ---------  "
+		    "------  ------\n");
 	}
 
 	*count += 1;
 
-	(void) printf("%3d  %-15s  %-15llu  %-15llu  %llx  %6lu\n", id, pool,
+	double freq = record->zi_freq == 0 ? 100.0f :
+	    (((double)record->zi_freq) / ZI_PERCENTAGE_MAX) * 100.0f;
+
+	(void) printf("%3d  %-15s  %llx  %10llu  %5llu  %8.4f%%  "
+	    "%6lu  %6lu\n", id, pool, (u_longlong_t)record->zi_guid,
 	    (u_longlong_t)NSEC2MSEC(record->zi_timer),
 	    (u_longlong_t)record->zi_nlanes,
-	    (u_longlong_t)record->zi_guid,
-	    record->zi_inject_count);
+	    freq, record->zi_match_count, record->zi_inject_count);
 
 	return (0);
 }

module/zfs/zio_inject.c

@@ -631,9 +631,6 @@ zio_handle_io_delay(zio_t *zio)
 		if (handler->zi_record.zi_cmd != ZINJECT_DELAY_IO)
 			continue;
 
-		if (!freq_triggered(handler->zi_record.zi_freq))
-			continue;
-
 		if (vd->vdev_guid != handler->zi_record.zi_guid)
 			continue;
 
@@ -658,6 +655,12 @@ zio_handle_io_delay(zio_t *zio)
 		ASSERT3U(handler->zi_record.zi_nlanes, >,
 		    handler->zi_next_lane);
 
+		handler->zi_record.zi_match_count++;
+
+		/* Limit the use of this handler if requested */
+		if (!freq_triggered(handler->zi_record.zi_freq))
+			continue;
+
 		/*
 		 * We want to issue this IO to the lane that will become
 		 * idle the soonest, so we compare the soonest this
@@ -730,7 +733,6 @@ zio_handle_io_delay(zio_t *zio)
 		min_handler->zi_next_lane = (min_handler->zi_next_lane + 1) %
 		    min_handler->zi_record.zi_nlanes;
 
-		min_handler->zi_record.zi_match_count++;
 		min_handler->zi_record.zi_inject_count++;
 
 	}

tests/zfs-tests/tests/functional/cli_root/zinject/zinject_counts.ksh

@@ -116,20 +116,19 @@ function test_object_injection
 	done
 }
 
-# Test delay injections, by injecting delays and writing. Note that there
-# is no frequency option for these, and match counts are not reqported, so
-# we only look for the inject count in the last column.
+# Test delay injections, by injecting delays and writing
 function test_delay_injection
 {
-	log_must zinject -d $DISK1 -D 50:1 $TESTPOOL
+	for freq in 100 50 ; do
+		log_must zinject -d $DISK1 -D 50:1 -f $freq $TESTPOOL
 
-	log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1
-	zpool sync
+		log_must dd if=/dev/urandom of=/$TESTPOOL/file bs=1M count=1
+		zpool sync
 
-	typeset -i match=($(zinject | grep -oE ' [0-9]+$'))
-	log_must test $match -gt 0
+		log_must check_count_freq $freq
 
-	log_must zinject -c all
+		log_must zinject -c all
+	done
 }
 
 # Disable cache, to ensure reads induce IO