Update arc_available_memory() to check freemem

While Linux doesn't provide detailed information about the state of
the VM it does provide us total free pages.  This information should
be incorporated in to the arc_available_memory() calculation rather
than solely relying on a signal from direct reclaim.

It is also desirable that the amount of reclaim be tunable on a
target system.  While the default values are expected to work well
for most workloads there may be cases where custom values are needed.

zfs_arc_lotsfree - Threshold in bytes for what the ARC should consider
                   to be a lot of free memory on the system.

zfs_arc_desfree  - Threshold in bytes for what the ARC should consider
                   to be the desired available free memory on the system.

Note that zfs_arc_lotsfree and zfs_arc_desfree are defined in terms
of bytes unlike the illumos globals lotsfree and desfree.  This was
done to make reading and setting the values easier.  The current values
are available in /proc/spl/kstat/zfs/arcstats.

Signed-off-by: Brian Behlendorf <[email protected]>
Issue #3637

man/man5/zfs-module-parameters.5

@@ -385,6 +385,19 @@ Seconds before growing arc size
 Default value: \fB5\fR.
 .RE
 
+.sp
+.ne 2
+.na
+\fBzfs_arc_lotsfree\fR (int)
+.ad
+.RS 12n
+Threshold in bytes used by the ARC to determine if there is a large amount
+of free memory on the system.  Defaults to the larger of 1/64 of physical
+memory or 512K but may be overridden by specifying \fBzfs_arc_lotsfree\fR.
+.sp
+Default value: \fB0\fR.
+.RE
+
 .sp
 .ne 2
 .na
@@ -476,6 +489,19 @@ Min arc size
 Default value: \fB100\fR.
 .RE
 
+.sp
+.ne 2
+.na
+\fBzfs_arc_desfree\fR (int)
+.ad
+.RS 12n
+Threshold in bytes used by the ARC to determine the desired amount of free
+memory on the system.  Defaults to the larger of \fBzfs_arc_lotsfree\fR/2
+or 200K but may be overridden by specifying \fBzfs_arc_desfree\fR.
+.sp
+Default value: \fB0\fR.
+.RE
+
 .sp
 .ne 2
 .na

module/zfs/arc.c

@@ -247,6 +247,8 @@ int zfs_arc_p_dampener_disable = 1;
 int zfs_arc_meta_prune = 10000;
 int zfs_arc_meta_strategy = ARC_STRATEGY_META_BALANCED;
 int zfs_arc_meta_adjust_restarts = 4096;
+unsigned long zfs_arc_lotsfree = 0;
+unsigned long zfs_arc_desfree = 0;
 
 /* The 6 states: */
 static arc_state_t ARC_anon;
@@ -473,6 +475,9 @@ typedef struct arc_stats {
 	kstat_named_t arcstat_meta_limit;
 	kstat_named_t arcstat_meta_max;
 	kstat_named_t arcstat_meta_min;
+	kstat_named_t arcstat_needfree;
+	kstat_named_t arcstat_lotsfree;
+	kstat_named_t arcstat_desfree;
 } arc_stats_t;
 
 static arc_stats_t arc_stats = {
@@ -564,7 +569,10 @@ static arc_stats_t arc_stats = {
 	{ "arc_meta_used",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_limit",		KSTAT_DATA_UINT64 },
 	{ "arc_meta_max",		KSTAT_DATA_UINT64 },
-	{ "arc_meta_min",		KSTAT_DATA_UINT64 }
+	{ "arc_meta_min",		KSTAT_DATA_UINT64 },
+	{ "arc_needfree",		KSTAT_DATA_UINT64 },
+	{ "arc_lotsfree",		KSTAT_DATA_UINT64 },
+	{ "arc_desfree",		KSTAT_DATA_UINT64 }
 };
 
 #define	ARCSTAT(stat)	(arc_stats.stat.value.ui64)
@@ -633,6 +641,9 @@ static arc_state_t	*arc_l2c_only;
 #define	arc_meta_min	ARCSTAT(arcstat_meta_min) /* min size for metadata */
 #define	arc_meta_used	ARCSTAT(arcstat_meta_used) /* size of metadata */
 #define	arc_meta_max	ARCSTAT(arcstat_meta_max) /* max size of metadata */
+#define	arc_needfree	ARCSTAT(arcstat_needfree) /* bytes needed to be freed */
+#define	arc_lotsfree	ARCSTAT(arcstat_lotsfree) /* lots of free bytes */
+#define	arc_desfree	ARCSTAT(arcstat_desfree) /* desired free bytes */
 
 #define	L2ARC_IS_VALID_COMPRESS(_c_) \
 	((_c_) == ZIO_COMPRESS_LZ4 || (_c_) == ZIO_COMPRESS_EMPTY)
@@ -3222,12 +3233,6 @@ int64_t last_free_memory;
 free_memory_reason_t last_free_reason;
 
 #ifdef _KERNEL
-#ifdef __linux__
-/*
- * expiration time for arc_no_grow set by direct memory reclaim.
- */
-static clock_t arc_grow_time = 0;
-#else
 /*
  * Additional reserve of pages for pp_reserve.
  */
@@ -3237,7 +3242,6 @@ int64_t arc_pages_pp_reserve = 64;
  * Additional reserve of pages for swapfs.
  */
 int64_t arc_swapfs_reserve = 64;
-#endif
 #endif /* _KERNEL */
 
 /*
@@ -3250,26 +3254,14 @@ arc_available_memory(void)
 {
 	int64_t lowest = INT64_MAX;
 	free_memory_reason_t r = FMR_UNKNOWN;
-
 #ifdef _KERNEL
-#ifdef __linux__
-	/*
-	 * Under Linux we are not allowed to directly interrogate the global
-	 * memory state.  Instead rely on observing that direct reclaim has
-	 * recently occurred therefore the system must be low on memory.  The
-	 * exact values returned are not critical but should be small.
-	 */
-	if (ddi_time_after_eq(ddi_get_lbolt(), arc_grow_time))
-		lowest = PAGE_SIZE;
-	else
-		lowest = -PAGE_SIZE;
-#else
 	int64_t n;
+#ifdef __linux__
+	pgcnt_t needfree = btop(arc_needfree);
+	pgcnt_t lotsfree = btop(arc_lotsfree);
+	pgcnt_t desfree = btop(arc_desfree);
+#endif
 
-	/*
-	 * Platforms like illumos have greater visibility in to the memory
-	 * subsystem and can return a more detailed analysis of memory.
-	 */
 	if (needfree > 0) {
 		n = PAGESIZE * (-needfree);
 		if (n < lowest) {
@@ -3291,6 +3283,7 @@ arc_available_memory(void)
 		r = FMR_LOTSFREE;
 	}
 
+#ifndef __linux__
 	/*
 	 * check to make sure that swapfs has enough space so that anon
 	 * reservations can still succeed. anon_resvmem() checks that the
@@ -3319,6 +3312,7 @@ arc_available_memory(void)
 		lowest = n;
 		r = FMR_PAGES_PP_MAXIMUM;
 	}
+#endif
 
 #if defined(__i386)
 	/*
@@ -3357,7 +3351,6 @@ arc_available_memory(void)
 			r = FMR_ZIO_ARENA;
 		}
 	}
-#endif /* __linux__ */
 #else
 	/* Every 100 calls, free a small amount */
 	if (spa_get_random(100) == 0)
@@ -3480,7 +3473,7 @@ arc_reclaim_thread(void)
 			to_free = (arc_c >> arc_shrink_shift) - free_memory;
 			if (to_free > 0) {
 #ifdef _KERNEL
-				to_free = MAX(to_free, ptob(needfree));
+				to_free = MAX(to_free, arc_needfree);
 #endif
 				arc_shrink(to_free);
 			}
@@ -3507,9 +3500,11 @@ arc_reclaim_thread(void)
 			/*
 			 * We're either no longer overflowing, or we
 			 * can't evict anything more, so we should wake
-			 * up any threads before we go to sleep.
+			 * up any threads before we go to sleep and clear
+			 * arc_needfree since nothing more can be done.
 			 */
 			cv_broadcast(&arc_reclaim_waiters_cv);
+			arc_needfree = 0;
 
 			/*
 			 * Block until signaled, or after one second (we
@@ -3713,7 +3708,7 @@ __arc_shrinker_func(struct shrinker *shrink, struct shrink_control *sc)
 		ARCSTAT_BUMP(arcstat_memory_indirect_count);
 	} else {
 		arc_no_grow = B_TRUE;
-		arc_grow_time = ddi_get_lbolt() + (zfs_arc_grow_retry * hz);
+		arc_needfree = ptob(sc->nr_to_scan);
 		ARCSTAT_BUMP(arcstat_memory_direct_count);
 	}
 
@@ -5288,6 +5283,14 @@ arc_tuning_update(void)
 	/* Valid range: 1 - N ticks */
 	if (zfs_arc_min_prefetch_lifespan)
 		arc_min_prefetch_lifespan = zfs_arc_min_prefetch_lifespan;
+
+	/* Valid range: 0 - <all physical memory> */
+	if ((zfs_arc_lotsfree) && (zfs_arc_lotsfree != arc_lotsfree))
+		arc_lotsfree = MIN(MAX(zfs_arc_lotsfree, 0), ptob(physmem));
+
+	/* Valid range: 0 - <all physical memory> */
+	if ((zfs_arc_desfree) && (zfs_arc_desfree != arc_desfree))
+		arc_desfree = MIN(MAX(zfs_arc_desfree, 0), ptob(physmem));
 }
 
 void
@@ -5329,6 +5332,14 @@ arc_init(void)
 	 * swapping out pages when it is preferable to shrink the arc.
 	 */
 	spl_register_shrinker(&arc_shrinker);
+
+	/*
+	 * Thresholds for assessing low memory situation.  The default
+	 * values are the same as on other platforms such as illumos.
+	 */
+	arc_needfree = 0;
+	arc_lotsfree = MAX(ptob(physmem / 64), (512 * 1024));
+	arc_desfree = MAX(arc_lotsfree / 2, (200 * 1024));
 #endif
 
 	/* Set min cache to allow safe operation of arc_adapt() */
@@ -7064,4 +7075,10 @@ MODULE_PARM_DESC(l2arc_feed_again, "Turbo L2ARC warmup");
 module_param(l2arc_norw, int, 0644);
 MODULE_PARM_DESC(l2arc_norw, "No reads during writes");
 
+module_param(zfs_arc_lotsfree, ulong, 0644);
+MODULE_PARM_DESC(zfs_arc_lotsfree, "Threshold defining lots of free memory");
+
+module_param(zfs_arc_desfree, ulong, 0644);
+MODULE_PARM_DESC(zfs_arc_desfree, "Threshold defining desired free memory");
+
 #endif