Balance vdev mirror reads off pending queue length

When selecting which vdev mirror child to read from,
redirect io's to child vdevs with the smallest pending
queue lengths.

During the vdev selection process if a readable vdev is found
with a queue length of 0 then the vdev will be used and the
above process will be cut short.

It is hoped that this will cause all available vdevs to be
utilised while ensuring the more capable devices take on
more of the workload.

closes openzfs#1461

include/sys/vdev.h

@@ -69,6 +69,7 @@ extern void vdev_dtl_reassess(vdev_t *vd, uint64_t txg, uint64_t scrub_txg,
 extern boolean_t vdev_dtl_required(vdev_t *vd);
 extern boolean_t vdev_resilver_needed(vdev_t *vd,
     uint64_t *minp, uint64_t *maxp);
+extern int vdev_pending_queued(vdev_t *vd);
 
 extern void vdev_hold(vdev_t *);
 extern void vdev_rele(vdev_t *);

module/zfs/vdev.c

@@ -1948,6 +1948,20 @@ vdev_resilver_needed(vdev_t *vd, uint64_t *minp, uint64_t *maxp)
 	return (needed);
 }
 
+int
+vdev_pending_queued(vdev_t *vd)
+{
+	int pending;
+
+	vdev_queue_t *vq = &vd->vdev_queue;
+
+	mutex_enter(&vq->vq_lock);
+	pending = avl_numnodes(&vq->vq_pending_tree);
+	mutex_exit(&vq->vq_lock);
+
+	return pending;
+}
+
 void
 vdev_load(vdev_t *vd)
 {

module/zfs/vdev_mirror.c

@@ -221,7 +221,9 @@ vdev_mirror_child_select(zio_t *zio)
 	mirror_map_t *mm = zio->io_vsd;
 	mirror_child_t *mc;
 	uint64_t txg = zio->io_txg;
-	int i, c;
+	int pending_lowest_child = -1;
+	int pending_lowest_count = INT_MAX;
+	int i, c, pending;
 
 	ASSERT(zio->io_bp == NULL || BP_PHYSICAL_BIRTH(zio->io_bp) == txg);
 
@@ -243,12 +245,28 @@ vdev_mirror_child_select(zio_t *zio)
 			continue;
 		}
 		if (!vdev_dtl_contains(mc->mc_vd, DTL_MISSING, txg, 1))
-			return (c);
+		{
+			pending = vdev_pending_queued(mc->mc_vd);
+			if (pending == 0)
+				return (c);
+			if (pending < pending_lowest_count) {
+				pending_lowest_count = pending;
+				pending_lowest_child = c;
+			}
+			continue;
+		}
 		mc->mc_error = ESTALE;
 		mc->mc_skipped = 1;
 		mc->mc_speculative = 1;
 	}
 
+	/*
+	 * See if we found multiple devices with pending io's
+	 * and return the child with smallest queue.
+	 */
+	if ( pending_lowest_child != -1 )
+		return (pending_lowest_child);
+
 	/*
 	 * Every device is either missing or has this txg in its DTL.
 	 * Look for any child we haven't already tried before giving up.