DAOS-13701: Memory bucket allocator API definition

- umem routines that allocates memory block will now take memory bucket id as and additional argument. umem internally now calls the extended dav allocator routines and memory bucket id is passed embedded in flags. - umem_get_mb_evictable() and dav_get_zone_evictable() are added to support allocator returning preferred zone to be used as evictable memory bucket for current allocations. Right now these routines always return zero. - The dav heap runtime is cleaned up to make provision for memory bucket implementation. Signed-off-by: Sherin T George <[email protected]>
daos-stack · Sep 21, 2023 · 2b3bc9d · 2b3bc9d
1 parent f7fe80f
commit 2b3bc9d
Show file tree

Hide file tree

Showing 33 changed files with 594 additions and 1,078 deletions.
diff --git a/src/common/ad_tx.c b/src/common/ad_tx.c
@@ -1148,7 +1148,7 @@ umo_tx_free(struct umem_instance *umm, umem_off_t umoff)
 
 static umem_off_t
 umo_tx_alloc(struct umem_instance *umm, size_t size, uint64_t flags,
-	     unsigned int type_num)
+	     unsigned int type_num, unsigned int mbkt_id)
 {
 	struct ad_tx		*tx = tx_get();
 	struct ad_blob_handle	 bh = umm2ad_blob_hdl(umm);
@@ -1242,7 +1242,8 @@ umo_tx_add_ptr(struct umem_instance *umm, void *ptr, size_t size)
 }
 
 static umem_off_t
-umo_reserve(struct umem_instance *umm, void *act, size_t size, unsigned int type_num)
+umo_reserve(struct umem_instance *umm, void *act, size_t size, unsigned int type_num,
+	    unsigned int mbkt_id)
 {
 	struct ad_blob_handle	 bh = umm2ad_blob_hdl(umm);
 	struct ad_reserv_act	*ract = act;
@@ -1330,9 +1331,10 @@ umo_atomic_copy(struct umem_instance *umm, void *dest, const void *src, size_t l
 }
 
 static umem_off_t
-umo_atomic_alloc(struct umem_instance *umm, size_t size, unsigned int type_num)
+umo_atomic_alloc(struct umem_instance *umm, size_t size, unsigned int type_num,
+		 unsigned int mbkt_id)
 {
-	return umo_tx_alloc(umm, size, 0, type_num);
+	return umo_tx_alloc(umm, size, 0, type_num, mbkt_id);
 }
 
 static int

diff --git a/src/common/btree.c b/src/common/btree.c
@@ -667,7 +667,7 @@ btr_node_alloc(struct btr_context *tcx, umem_off_t *nd_off_p)
 		nd_off = btr_ops(tcx)->to_node_alloc(&tcx->tc_tins,
 						     btr_node_size(tcx));
 	else
-		nd_off = umem_zalloc(btr_umm(tcx), btr_node_size(tcx));
+		nd_off = umem_zalloc(btr_umm(tcx), btr_node_size(tcx), 0);
 
 	if (UMOFF_IS_NULL(nd_off))
 		return btr_umm(tcx)->umm_nospc_rc;
@@ -863,7 +863,7 @@ btr_root_alloc(struct btr_context *tcx)
 	struct btr_root		*root;
 
 	tins->ti_root_off = umem_zalloc(btr_umm(tcx),
-					sizeof(struct btr_root));
+					sizeof(struct btr_root), 0);
 	if (UMOFF_IS_NULL(tins->ti_root_off))
 		return btr_umm(tcx)->umm_nospc_rc;
 

diff --git a/src/common/btree_class.c b/src/common/btree_class.c
@@ -248,15 +248,15 @@ nv_rec_alloc(struct btr_instance *tins, d_iov_t *key, d_iov_t *val,
 
 	rc = tins->ti_umm.umm_nospc_rc;
 
-	roff = umem_zalloc(&tins->ti_umm, sizeof(*r) + name_len);
+	roff = umem_zalloc(&tins->ti_umm, sizeof(*r) + name_len, 0);
 	if (UMOFF_IS_NULL(roff))
 		D_GOTO(err, rc = tins->ti_umm.umm_nospc_rc);
 
 	r = umem_off2ptr(&tins->ti_umm, roff);
 	r->nr_value_size = val->iov_len;
 	r->nr_value_buf_size = r->nr_value_size;
 
-	r->nr_value = umem_alloc(&tins->ti_umm, r->nr_value_buf_size);
+	r->nr_value = umem_alloc(&tins->ti_umm, r->nr_value_buf_size, 0);
 	if (UMOFF_IS_NULL(r->nr_value))
 		D_GOTO(err_r, rc = tins->ti_umm.umm_nospc_rc);
 
@@ -332,7 +332,7 @@ nv_rec_update(struct btr_instance *tins, struct btr_record *rec,
 	if (r->nr_value_buf_size < val->iov_len) {
 		umem_off_t voff;
 
-		voff = umem_alloc(&tins->ti_umm, val->iov_len);
+		voff = umem_alloc(&tins->ti_umm, val->iov_len, 0);
 		if (UMOFF_IS_NULL(voff))
 			return tins->ti_umm.umm_nospc_rc;
 
@@ -588,15 +588,15 @@ uv_rec_alloc(struct btr_instance *tins, d_iov_t *key, d_iov_t *val,
 	    val->iov_len == 0 || val->iov_buf_len < val->iov_len)
 		D_GOTO(err, rc);
 
-	roff = umem_zalloc(&tins->ti_umm, sizeof(*r));
+	roff = umem_zalloc(&tins->ti_umm, sizeof(*r), 0);
 	if (UMOFF_IS_NULL(roff))
 		D_GOTO(err, rc = tins->ti_umm.umm_nospc_rc);
 
 	r = umem_off2ptr(&tins->ti_umm, roff);
 	r->ur_value_size = val->iov_len;
 	r->ur_value_buf_size = r->ur_value_size;
 
-	r->ur_value = umem_alloc(&tins->ti_umm, r->ur_value_buf_size);
+	r->ur_value = umem_alloc(&tins->ti_umm, r->ur_value_buf_size, 0);
 	if (UMOFF_IS_NULL(r->ur_value))
 		D_GOTO(err_r, rc = tins->ti_umm.umm_nospc_rc);
 
@@ -670,7 +670,7 @@ uv_rec_update(struct btr_instance *tins, struct btr_record *rec,
 	if (r->ur_value_buf_size < val->iov_len) {
 		umem_off_t voff;
 
-		voff = umem_alloc(&tins->ti_umm, val->iov_len);
+		voff = umem_alloc(&tins->ti_umm, val->iov_len, 0);
 		if (UMOFF_IS_NULL(voff))
 			return tins->ti_umm.umm_nospc_rc;
 
@@ -907,7 +907,7 @@ ec_rec_alloc(struct btr_instance *tins, d_iov_t *key, d_iov_t *val,
 	    val->iov_buf_len < val->iov_len)
 		return rc;
 
-	roff = umem_zalloc(&tins->ti_umm, sizeof(*r));
+	roff = umem_zalloc(&tins->ti_umm, sizeof(*r), 0);
 	if (UMOFF_IS_NULL(roff))
 		return tins->ti_umm.umm_nospc_rc;
 
@@ -1135,14 +1135,14 @@ kv_rec_alloc(struct btr_instance *tins, d_iov_t *key, d_iov_t *val,
 	    val->iov_buf_len < val->iov_len)
 		D_GOTO(err, rc = -DER_INVAL);
 
-	roff = umem_zalloc(&tins->ti_umm, sizeof(*r) + key->iov_len);
+	roff = umem_zalloc(&tins->ti_umm, sizeof(*r) + key->iov_len, 0);
 	if (UMOFF_IS_NULL(roff))
 		D_GOTO(err, rc = tins->ti_umm.umm_nospc_rc);
 	r = umem_off2ptr(&tins->ti_umm, roff);
 
 	r->kr_value_len = val->iov_len;
 	r->kr_value_cap = r->kr_value_len;
-	r->kr_value = umem_alloc(&tins->ti_umm, r->kr_value_cap);
+	r->kr_value = umem_alloc(&tins->ti_umm, r->kr_value_cap, 0);
 	if (UMOFF_IS_NULL(r->kr_value))
 		D_GOTO(err_r, rc = tins->ti_umm.umm_nospc_rc);
 	v = umem_off2ptr(&tins->ti_umm, r->kr_value);
@@ -1210,7 +1210,7 @@ kv_rec_update(struct btr_instance *tins, struct btr_record *rec,
 	if (r->kr_value_cap < val->iov_len) {
 		umem_off_t voff;
 
-		voff = umem_alloc(&tins->ti_umm, val->iov_len);
+		voff = umem_alloc(&tins->ti_umm, val->iov_len, 0);
 		if (UMOFF_IS_NULL(voff))
 			return tins->ti_umm.umm_nospc_rc;
 		umem_free(&tins->ti_umm, r->kr_value);
@@ -1330,14 +1330,14 @@ iv_rec_alloc(struct btr_instance *tins, d_iov_t *key, d_iov_t *val,
 	    key->iov_buf_len < key->iov_len || val->iov_buf_len < val->iov_len)
 		D_GOTO(err, rc = -DER_INVAL);
 
-	roff = umem_zalloc(&tins->ti_umm, sizeof(*r));
+	roff = umem_zalloc(&tins->ti_umm, sizeof(*r), 0);
 	if (UMOFF_IS_NULL(roff))
 		D_GOTO(err, rc = tins->ti_umm.umm_nospc_rc);
 	r = umem_off2ptr(&tins->ti_umm, roff);
 
 	r->ir_value_len = val->iov_len;
 	r->ir_value_cap = r->ir_value_len;
-	r->ir_value = umem_alloc(&tins->ti_umm, r->ir_value_cap);
+	r->ir_value = umem_alloc(&tins->ti_umm, r->ir_value_cap, 0);
 	if (UMOFF_IS_NULL(r->ir_value))
 		D_GOTO(err_r, rc = tins->ti_umm.umm_nospc_rc);
 	v = umem_off2ptr(&tins->ti_umm, r->ir_value);
@@ -1413,7 +1413,7 @@ iv_rec_update(struct btr_instance *tins, struct btr_record *rec,
 	if (r->ir_value_cap < val->iov_len) {
 		umem_off_t voff;
 
-		voff = umem_alloc(&tins->ti_umm, val->iov_len);
+		voff = umem_alloc(&tins->ti_umm, val->iov_len, 0);
 		if (UMOFF_IS_NULL(voff))
 			return tins->ti_umm.umm_nospc_rc;
 		umem_free(&tins->ti_umm, r->ir_value);
@@ -1520,7 +1520,7 @@ ifv_rec_alloc(struct btr_instance *tins, d_iov_t *key, d_iov_t *val, struct btr_
 	    val->iov_buf_len < val->iov_len)
 		D_GOTO(err, rc = -DER_INVAL);
 
-	roff = umem_zalloc(&tins->ti_umm, sizeof(*r) + val->iov_len);
+	roff = umem_zalloc(&tins->ti_umm, sizeof(*r) + val->iov_len, 0);
 	if (UMOFF_IS_NULL(roff))
 		D_GOTO(err, rc = tins->ti_umm.umm_nospc_rc);
 	r = umem_off2ptr(&tins->ti_umm, roff);

diff --git a/src/common/dav/bucket.c b/src/common/dav/bucket.c
@@ -23,15 +23,13 @@
 
 struct bucket {
 	/* this struct is both the lock guard and the locked state */
-	struct bucket_locked *locked;
-
-	struct alloc_class *aclass;
-
-	struct block_container *container;
+	struct bucket_locked             *locked;
+	struct alloc_class               *aclass;
+	struct block_container           *container;
 	const struct block_container_ops *c_ops;
-
-	struct memory_block_reserved *active_memory_block;
-	int is_active;
+	struct memory_block_reserved     *active_memory_block;
+	struct zone_set                  *zset;
+	int                               is_active;
 };
 
 struct bucket_locked {
@@ -77,7 +75,7 @@ bucket_fini(struct bucket *b)
  * bucket_locked_new -- creates a new locked bucket instance
  */
 struct bucket_locked *
-bucket_locked_new(struct block_container *c, struct alloc_class *aclass)
+bucket_locked_new(struct block_container *c, struct alloc_class *aclass, struct zone_set *zset)
 {
 	ASSERTne(c, NULL);
 
@@ -92,6 +90,7 @@ bucket_locked_new(struct block_container *c, struct alloc_class *aclass)
 
 	util_mutex_init(&b->lock);
 	b->bucket.locked = b;
+	b->bucket.zset = zset;
 
 	return b;
 
@@ -268,3 +267,9 @@ bucket_active_block(struct bucket *b)
 {
 	return b->is_active ? b->active_memory_block : NULL;
 }
+
+struct zone_set *
+bucket_get_zoneset(struct bucket *b)
+{
+	return b->zset;
+}
diff --git a/src/common/dav/bucket.h b/src/common/dav/bucket.h
@@ -22,7 +22,8 @@ struct bucket_locked;
 struct bucket;
 
 struct bucket_locked *bucket_locked_new(struct block_container *c,
-					struct alloc_class *aclass);
+					struct alloc_class *aclass,
+					struct zone_set *zset);
 
 struct bucket *bucket_acquire(struct bucket_locked *b);
 void bucket_release(struct bucket *b);
@@ -41,5 +42,6 @@ int bucket_detach_run(struct bucket *b,
 struct memory_block_reserved *bucket_active_block(struct bucket *b);
 
 void bucket_locked_delete(struct bucket_locked *b);
+struct zone_set *bucket_get_zoneset(struct bucket *b);
 
 #endif /* __DAOS_COMMON_BUCKET_H */
diff --git a/src/common/dav/dav.h b/src/common/dav/dav.h
@@ -23,17 +23,17 @@
 #define DAV_FLAG_TX_NO_ABORT		(((uint64_t)1) << 4)
 
 #define DAV_CLASS_ID(id)		(((uint64_t)(id)) << 48)
-#define DAV_ARENA_ID(id)		(((uint64_t)(id)) << 32)
+#define DAV_EZONE_ID(id)		(((uint64_t)(id)) << 16)
 
 #define DAV_XALLOC_CLASS_MASK		((((uint64_t)1 << 16) - 1) << 48)
-#define DAV_XALLOC_ARENA_MASK		((((uint64_t)1 << 16) - 1) << 32)
+#define DAV_XALLOC_EZONE_MASK		((((uint64_t)1 << 32) - 1) << 16)
 #define DAV_XALLOC_ZERO			DAV_FLAG_ZERO
 #define DAV_XALLOC_NO_FLUSH		DAV_FLAG_NO_FLUSH
 #define DAV_XALLOC_NO_ABORT		DAV_FLAG_TX_NO_ABORT
 
 #define DAV_TX_XALLOC_VALID_FLAGS	(DAV_XALLOC_ZERO |\
 					DAV_XALLOC_NO_FLUSH |\
-					DAV_XALLOC_ARENA_MASK |\
+					DAV_XALLOC_EZONE_MASK |\
 					DAV_XALLOC_CLASS_MASK |\
 					DAV_XALLOC_NO_ABORT)
 
@@ -125,6 +125,12 @@ typedef int (*dav_constr)(dav_obj_t *pop, void *ptr, void *arg);
 int dav_alloc(dav_obj_t *pop, uint64_t *offp, size_t size,
 	      uint64_t type_num, dav_constr constructor, void *arg);
 
+/*
+ * Allocates with flags a new object from the pool.
+ */
+int dav_xalloc(dav_obj_t *pop, uint64_t *offp, size_t size, uint64_t type_num,
+	       uint64_t flags, dav_constr constructor, void *arg);
+
 /**
  * Frees the memory at specified offset within the DAV object pointed to by hdl.
  *
@@ -370,7 +376,14 @@ struct dav_action {
 	};
 };
 
+#define DAV_ACTION_XRESERVE_VALID_FLAGS \
+	(DAV_XALLOC_CLASS_MASK |\
+	 DAV_XALLOC_EZONE_MASK |\
+	 DAV_XALLOC_ZERO)
+
 uint64_t dav_reserve(dav_obj_t *pop, struct dav_action *act, size_t size, uint64_t type_num);
+uint64_t dav_xreserve(dav_obj_t *pop, struct dav_action *act, size_t size, uint64_t type_num,
+		      uint64_t flags);
 void dav_defer_free(dav_obj_t *pop, uint64_t off, struct dav_action *act);
 int dav_publish(dav_obj_t *pop, struct dav_action *actv, size_t actvcnt);
 void dav_cancel(dav_obj_t *pop, struct dav_action *actv, size_t actvcnt);
@@ -518,4 +531,15 @@ uint32_t wal_tx_payload_len(struct umem_wal_tx *tx);
 struct umem_action *wal_tx_act_first(struct umem_wal_tx *tx);
 struct umem_action *wal_tx_act_next(struct umem_wal_tx *tx);
 
+/**
+ * Get an evictable zone with sufficient free space within.
+ *
+ * \param[in]		pop		pool handle
+ * \param[in]		flags		zone selection criteria.
+ *
+ * \return id >= 0. Zero indicates non-evictable zone and will be
+ *	returned if no evictable zone can be chosen.
+ */
+uint32_t dav_get_zone_evictable(dav_obj_t *pop, int flags);
+
 #endif /* __DAOS_COMMON_DAV_H */
diff --git a/src/common/dav/dav_iface.c b/src/common/dav/dav_iface.c
@@ -180,13 +180,6 @@ dav_obj_open_internal(int fd, int flags, size_t sz, const char *path, struct ume
 		palloc_heap_vg_open(hdl->do_heap, 1);
 #endif
 
-	rc = heap_buckets_init(hdl->do_heap);
-	if (rc) {
-		err = rc;
-		heap_cleanup(hdl->do_heap);
-		goto out2;
-	}
-
 	rc = dav_create_clogs(hdl);
 	if (rc) {
 		err = rc;

diff --git a/src/common/dav/dav_internal.h b/src/common/dav/dav_internal.h
@@ -28,11 +28,6 @@ enum dav_stats_enabled {
 	DAV_STATS_DISABLED,
 };
 
-enum dav_arenas_assignment_type {
-	DAV_ARENAS_ASSIGNMENT_THREAD_KEY,
-	DAV_ARENAS_ASSIGNMENT_GLOBAL,
-};
-
 #define	DAV_PHDR_SIZE	4096
 
 /* DAV header data that will be persisted */