Dynamic taskq simplification

There's no particular need for taskq_create() to spawn any threads for
a dynamic taskq so don't spawn any.

If the spl_taskq_thread_dynamic module parameter flag isn't set, disable
TASKQ_DYNAMIC if it's requested when a taskq is created.

Eliminate tqt_is_dynamic because tq_nspawn is only valid for dynamic
taskqs.

include/sys/taskq.h

@@ -118,10 +118,7 @@ typedef struct taskq_thread {
 	taskq_t			*tqt_tq;
 	taskqid_t		tqt_id;
 	taskq_ent_t		*tqt_task;
-	union {
-		uintptr_t	tqt_flags;
-		boolean_t	tqt_is_dynamic;
-	};
+	uintptr_t	tqt_flags;
 } taskq_thread_t;
 
 /* Global system-wide dynamic task queue available for all consumers */

module/spl/spl-taskq.c

@@ -53,7 +53,7 @@ EXPORT_SYMBOL(system_taskq);
 
 /* Private dedicated taskq for creating new taskq threads on demand. */
 static taskq_t *dynamic_taskq;
-static taskq_thread_t *taskq_thread_create(taskq_t *, boolean_t);
+static taskq_thread_t *taskq_thread_create(taskq_t *);
 
 /* List of all taskqs */
 LIST_HEAD(tq_list);
@@ -763,7 +763,7 @@ taskq_thread_spawn_task(void *arg)
 	taskq_t *tq = (taskq_t *)arg;
 	unsigned long flags;
 
-	if (taskq_thread_create(tq, B_TRUE) == NULL) {
+	if (taskq_thread_create(tq) == NULL) {
 		/* restore spawning count if failed */
 		spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 		tq->tq_nspawn--;
@@ -850,7 +850,7 @@ taskq_thread(void *args)
 	tsd_set(taskq_tsd, tq);
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
 	/* we are dynamically spawned, decrease spawning count */
-	if (tqt->tqt_is_dynamic)
+	if (tq->tq_flags & TASKQ_DYNAMIC)
 		tq->tq_nspawn--;
 
 	/* Immediately exit if more threads than allowed were created. */
@@ -961,7 +961,7 @@ taskq_thread(void *args)
 }
 
 static taskq_thread_t *
-taskq_thread_create(taskq_t *tq, boolean_t dynamic)
+taskq_thread_create(taskq_t *tq)
 {
 	static int last_used_cpu = 0;
 	taskq_thread_t *tqt;
@@ -971,7 +971,6 @@ taskq_thread_create(taskq_t *tq, boolean_t dynamic)
 	INIT_LIST_HEAD(&tqt->tqt_active_list);
 	tqt->tqt_tq = tq;
 	tqt->tqt_id = 0;
-	tqt->tqt_is_dynamic = dynamic;
 
 	tqt->tqt_thread = spl_kthread_create(taskq_thread, tqt,
 	    "%s", tq->tq_name);
@@ -1007,6 +1006,10 @@ taskq_create(const char *name, int nthreads, pri_t pri,
 	ASSERT(maxalloc <= INT_MAX);
 	ASSERT(!(flags & (TASKQ_CPR_SAFE))); /* Unsupported */
 
+	/* No need to flag as dynamic if the feature is disabled */
+	if (!spl_taskq_thread_dynamic)
+		flags &= ~(TASKQ_DYNAMIC);
+
 	/* Scale the number of threads using nthreads as a percentage */
 	if (flags & TASKQ_THREADS_CPU_PCT) {
 		ASSERT(nthreads <= 100);
@@ -1055,11 +1058,11 @@ taskq_create(const char *name, int nthreads, pri_t pri,
 		spin_unlock_irqrestore(&tq->tq_lock, irqflags);
 	}
 
-	if ((flags & TASKQ_DYNAMIC) && spl_taskq_thread_dynamic)
-		nthreads = 1;
+	if (flags & TASKQ_DYNAMIC)
+		nthreads = 0;
 
 	for (i = 0; i < nthreads; i++) {
-		tqt = taskq_thread_create(tq, B_FALSE);
+		tqt = taskq_thread_create(tq);
 		if (tqt == NULL)
 			rc = 1;
 		else
@@ -1111,11 +1114,15 @@ taskq_destroy(taskq_t *tq)
 	up_write(&tq_list_sem);
 
 	spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
+
 	/* wait for spawning threads to insert themselves to the list */
-	while (tq->tq_nspawn) {
-		spin_unlock_irqrestore(&tq->tq_lock, flags);
-		schedule_timeout_interruptible(1);
-		spin_lock_irqsave_nested(&tq->tq_lock, flags, tq->tq_lock_class);
+	if (tq->tq_flags & TASKQ_DYNAMIC) {
+		while (tq->tq_nspawn) {
+			spin_unlock_irqrestore(&tq->tq_lock, flags);
+			schedule_timeout_interruptible(1);
+			spin_lock_irqsave_nested(&tq->tq_lock, flags,
+			    tq->tq_lock_class);
+		}
 	}
 
 	/*