Partially revert "Add ddt, ddt_entry, and l2arc_hdr caches"

This reverts only the l2arc_hdr part of commit
ecf3d9b in preparation for the illumos
5497 "lock contention on arcs_mtx" patch which does the same thing
but uses the newer two-level ARC structure following the Illumos 5408
"managing ZFS cache devices requires lots of RAM" patch.

Signed-off-by: Tim Chase <[email protected]>
Signed-off-by: Brian Behlendorf <[email protected]>

module/zfs/arc.c

@@ -797,7 +797,6 @@ buf_hash_remove(arc_buf_hdr_t *buf)
  */
 static kmem_cache_t *hdr_cache;
 static kmem_cache_t *buf_cache;
-static kmem_cache_t *l2arc_hdr_cache;
 
 static void
 buf_fini(void)
@@ -819,7 +818,6 @@ buf_fini(void)
 		mutex_destroy(&buf_hash_table.ht_locks[i].ht_lock);
 	kmem_cache_destroy(hdr_cache);
 	kmem_cache_destroy(buf_cache);
-	kmem_cache_destroy(l2arc_hdr_cache);
 }
 
 /*
@@ -921,8 +919,6 @@ buf_init(void)
 	    0, hdr_cons, hdr_dest, NULL, NULL, NULL, 0);
 	buf_cache = kmem_cache_create("arc_buf_t", sizeof (arc_buf_t),
 	    0, buf_cons, buf_dest, NULL, NULL, NULL, 0);
-	l2arc_hdr_cache = kmem_cache_create("l2arc_buf_hdr_t", L2HDR_SIZE,
-	    0, NULL, NULL, NULL, NULL, NULL, 0);
 
 	for (i = 0; i < 256; i++)
 		for (ct = zfs_crc64_table + i, *ct = i, j = 8; j > 0; j--)
@@ -1598,7 +1594,7 @@ arc_hdr_destroy(arc_buf_hdr_t *hdr)
 			ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
 			vdev_space_update(l2hdr->b_dev->l2ad_vdev,
 			    -l2hdr->b_asize, 0, 0);
-			kmem_cache_free(l2arc_hdr_cache, l2hdr);
+			kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
 			arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
 			if (hdr->b_state == arc_l2c_only)
 				l2arc_hdr_stat_remove();
@@ -3738,7 +3734,7 @@ arc_release(arc_buf_t *buf, void *tag)
 		ARCSTAT_INCR(arcstat_l2_asize, -l2hdr->b_asize);
 		vdev_space_update(l2hdr->b_dev->l2ad_vdev,
 		    -l2hdr->b_asize, 0, 0);
-		kmem_cache_free(l2arc_hdr_cache, l2hdr);
+		kmem_free(l2hdr, sizeof (l2arc_buf_hdr_t));
 		arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
 		ARCSTAT_INCR(arcstat_l2_size, -buf_size);
 		mutex_exit(&l2arc_buflist_mtx);
@@ -4623,7 +4619,7 @@ l2arc_write_done(zio_t *zio)
 			ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
 			bytes_dropped += abl2->b_asize;
 			ab->b_l2hdr = NULL;
-			kmem_cache_free(l2arc_hdr_cache, abl2);
+			kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
 			arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
 			ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
 		}
@@ -4882,7 +4878,7 @@ l2arc_evict(l2arc_dev_t *dev, uint64_t distance, boolean_t all)
 				ARCSTAT_INCR(arcstat_l2_asize, -abl2->b_asize);
 				bytes_evicted += abl2->b_asize;
 				ab->b_l2hdr = NULL;
-				kmem_cache_free(l2arc_hdr_cache, abl2);
+				kmem_free(abl2, sizeof (l2arc_buf_hdr_t));
 				arc_space_return(L2HDR_SIZE, ARC_SPACE_L2HDRS);
 				ARCSTAT_INCR(arcstat_l2_size, -ab->b_size);
 			}
@@ -5028,9 +5024,9 @@ l2arc_write_buffers(spa_t *spa, l2arc_dev_t *dev, uint64_t target_sz,
 			/*
 			 * Create and add a new L2ARC header.
 			 */
-			l2hdr = kmem_cache_alloc(l2arc_hdr_cache, KM_SLEEP);
+			l2hdr = kmem_zalloc(sizeof (l2arc_buf_hdr_t),
+			    KM_PUSHPAGE);
 			l2hdr->b_dev = dev;
-			l2hdr->b_daddr = 0;
 			arc_space_consume(L2HDR_SIZE, ARC_SPACE_L2HDRS);
 
 			ab->b_flags |= ARC_L2_WRITING;