Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Optimize microzaps. #14039

Merged
merged 1 commit into from
Oct 20, 2022
Merged

Optimize microzaps. #14039

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions include/sys/btree.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ extern "C" {
* them, and increased memory overhead. Increasing these values results in
* higher variance in operation time, and reduces memory overhead.
*/
#define BTREE_CORE_ELEMS 128
#define BTREE_CORE_ELEMS 126
#define BTREE_LEAF_SIZE 4096

extern kmem_cache_t *zfs_btree_leaf_cache;
Expand Down Expand Up @@ -95,9 +95,6 @@ typedef struct zfs_btree_leaf {
uint8_t btl_elems[];
} zfs_btree_leaf_t;

#define BTREE_LEAF_ESIZE (BTREE_LEAF_SIZE - \
offsetof(zfs_btree_leaf_t, btl_elems))

typedef struct zfs_btree_index {
zfs_btree_hdr_t *bti_node;
uint32_t bti_offset;
Expand All @@ -109,14 +106,15 @@ typedef struct zfs_btree_index {
} zfs_btree_index_t;

typedef struct btree {
zfs_btree_hdr_t *bt_root;
int64_t bt_height;
int (*bt_compar) (const void *, const void *);
size_t bt_elem_size;
size_t bt_leaf_size;
uint32_t bt_leaf_cap;
int32_t bt_height;
uint64_t bt_num_elems;
uint64_t bt_num_nodes;
zfs_btree_hdr_t *bt_root;
zfs_btree_leaf_t *bt_bulk; // non-null if bulk loading
int (*bt_compar) (const void *, const void *);
} zfs_btree_t;

/*
Expand All @@ -132,9 +130,12 @@ void zfs_btree_fini(void);
* compar - function to compare two nodes, it must return exactly: -1, 0, or +1
* -1 for <, 0 for ==, and +1 for >
* size - the value of sizeof(struct my_type)
* lsize - custom leaf size
*/
void zfs_btree_create(zfs_btree_t *, int (*) (const void *, const void *),
size_t);
void zfs_btree_create_custom(zfs_btree_t *, int (*)(const void *, const void *),
size_t, size_t);

/*
* Find a node with a matching value in the tree. Returns the matching node
Expand Down
11 changes: 5 additions & 6 deletions include/sys/zap_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,10 +66,9 @@ typedef struct mzap_phys {
} mzap_phys_t;

typedef struct mzap_ent {
avl_node_t mze_node;
int mze_chunkid;
uint64_t mze_hash;
uint32_t mze_cd; /* copy from mze_phys->mze_cd */
uint32_t mze_hash;
uint16_t mze_cd; /* copy from mze_phys->mze_cd */
uint16_t mze_chunkid;
} mzap_ent_t;

#define MZE_PHYS(zap, mze) \
Expand Down Expand Up @@ -164,7 +163,7 @@ typedef struct zap {
int16_t zap_num_entries;
int16_t zap_num_chunks;
int16_t zap_alloc_next;
avl_tree_t zap_avl;
zfs_btree_t zap_tree;
} zap_micro;
} zap_u;
} zap_t;
Expand Down Expand Up @@ -203,7 +202,7 @@ int zap_lockdir(objset_t *os, uint64_t obj, dmu_tx_t *tx,
zap_t **zapp);
void zap_unlockdir(zap_t *zap, const void *tag);
void zap_evict_sync(void *dbu);
zap_name_t *zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt);
zap_name_t *zap_name_alloc_str(zap_t *zap, const char *key, matchtype_t mt);
void zap_name_free(zap_name_t *zn);
int zap_hashbits(zap_t *zap);
uint32_t zap_maxcd(zap_t *zap);
Expand Down
50 changes: 39 additions & 11 deletions module/zfs/btree.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,7 +102,7 @@ zfs_btree_poison_node(zfs_btree_t *tree, zfs_btree_hdr_t *hdr)
(void) memset(leaf->btl_elems, 0x0f, hdr->bth_first * size);
(void) memset(leaf->btl_elems +
(hdr->bth_first + hdr->bth_count) * size, 0x0f,
BTREE_LEAF_ESIZE -
tree->bt_leaf_size - offsetof(zfs_btree_leaf_t, btl_elems) -
(hdr->bth_first + hdr->bth_count) * size);
}
#endif
Expand Down Expand Up @@ -173,16 +173,44 @@ zfs_btree_fini(void)
kmem_cache_destroy(zfs_btree_leaf_cache);
}

static void *
zfs_btree_leaf_alloc(zfs_btree_t *tree)
{
if (tree->bt_leaf_size == BTREE_LEAF_SIZE)
return (kmem_cache_alloc(zfs_btree_leaf_cache, KM_SLEEP));
else
return (kmem_alloc(tree->bt_leaf_size, KM_SLEEP));
}

static void
zfs_btree_leaf_free(zfs_btree_t *tree, void *ptr)
{
if (tree->bt_leaf_size == BTREE_LEAF_SIZE)
return (kmem_cache_free(zfs_btree_leaf_cache, ptr));
else
return (kmem_free(ptr, tree->bt_leaf_size));
}

void
zfs_btree_create(zfs_btree_t *tree, int (*compar) (const void *, const void *),
size_t size)
{
ASSERT3U(size, <=, BTREE_LEAF_ESIZE / 2);
zfs_btree_create_custom(tree, compar, size, BTREE_LEAF_SIZE);
}

void
zfs_btree_create_custom(zfs_btree_t *tree,
int (*compar) (const void *, const void *),
size_t size, size_t lsize)
{
size_t esize = lsize - offsetof(zfs_btree_leaf_t, btl_elems);

ASSERT3U(size, <=, esize / 2);
memset(tree, 0, sizeof (*tree));
tree->bt_compar = compar;
tree->bt_elem_size = size;
tree->bt_leaf_cap = P2ALIGN(BTREE_LEAF_ESIZE / size, 2);
tree->bt_leaf_size = lsize;
tree->bt_leaf_cap = P2ALIGN(esize / size, 2);
tree->bt_height = -1;
tree->bt_bulk = NULL;
}
Expand Down Expand Up @@ -290,7 +318,7 @@ zfs_btree_find(zfs_btree_t *tree, const void *value, zfs_btree_index_t *where)

zfs_btree_core_t *node = NULL;
uint32_t child = 0;
uint64_t depth = 0;
uint32_t depth = 0;

/*
* Iterate down the tree, finding which child the value should be in
Expand Down Expand Up @@ -811,8 +839,7 @@ zfs_btree_insert_into_leaf(zfs_btree_t *tree, zfs_btree_leaf_t *leaf,
move_count++;
}
tree->bt_num_nodes++;
zfs_btree_leaf_t *new_leaf = kmem_cache_alloc(zfs_btree_leaf_cache,
KM_SLEEP);
zfs_btree_leaf_t *new_leaf = zfs_btree_leaf_alloc(tree);
zfs_btree_hdr_t *new_hdr = &new_leaf->btl_hdr;
new_hdr->bth_parent = leaf->btl_hdr.bth_parent;
new_hdr->bth_first = (tree->bt_bulk ? 0 : capacity / 4) +
Expand Down Expand Up @@ -1078,8 +1105,7 @@ zfs_btree_add_idx(zfs_btree_t *tree, const void *value,
ASSERT0(where->bti_offset);

tree->bt_num_nodes++;
zfs_btree_leaf_t *leaf = kmem_cache_alloc(zfs_btree_leaf_cache,
KM_SLEEP);
zfs_btree_leaf_t *leaf = zfs_btree_leaf_alloc(tree);
tree->bt_root = &leaf->btl_hdr;
tree->bt_height++;

Expand Down Expand Up @@ -1378,7 +1404,7 @@ zfs_btree_node_destroy(zfs_btree_t *tree, zfs_btree_hdr_t *node)
{
tree->bt_num_nodes--;
if (!zfs_btree_is_core(node)) {
kmem_cache_free(zfs_btree_leaf_cache, node);
zfs_btree_leaf_free(tree, node);
} else {
kmem_free(node, sizeof (zfs_btree_core_t) +
BTREE_CORE_ELEMS * tree->bt_elem_size);
Expand Down Expand Up @@ -1991,7 +2017,7 @@ zfs_btree_verify_counts(zfs_btree_t *tree)
*/
static uint64_t
zfs_btree_verify_height_helper(zfs_btree_t *tree, zfs_btree_hdr_t *hdr,
int64_t height)
int32_t height)
{
if (!zfs_btree_is_core(hdr)) {
VERIFY0(height);
Expand Down Expand Up @@ -2117,8 +2143,10 @@ zfs_btree_verify_poison_helper(zfs_btree_t *tree, zfs_btree_hdr_t *hdr)
zfs_btree_leaf_t *leaf = (zfs_btree_leaf_t *)hdr;
for (size_t i = 0; i < hdr->bth_first * size; i++)
VERIFY3U(leaf->btl_elems[i], ==, 0x0f);
size_t esize = tree->bt_leaf_size -
offsetof(zfs_btree_leaf_t, btl_elems);
for (size_t i = (hdr->bth_first + hdr->bth_count) * size;
i < BTREE_LEAF_ESIZE; i++)
i < esize; i++)
VERIFY3U(leaf->btl_elems[i], ==, 0x0f);
} else {
zfs_btree_core_t *node = (zfs_btree_core_t *)hdr;
Expand Down
4 changes: 2 additions & 2 deletions module/zfs/zap_leaf.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -646,7 +646,7 @@ zap_entry_create(zap_leaf_t *l, zap_name_t *zn, uint32_t cd,
* form of the name. But all callers have one of these on hand anyway,
* so might as well take advantage. A cleaner but slower interface
* would accept neither argument, and compute the normalized name as
* needed (using zap_name_alloc(zap_entry_read_name(zeh))).
* needed (using zap_name_alloc_str(zap_entry_read_name(zeh))).
*/
boolean_t
zap_entry_normalization_conflict(zap_entry_handle_t *zeh, zap_name_t *zn,
Expand All @@ -667,7 +667,7 @@ zap_entry_normalization_conflict(zap_entry_handle_t *zeh, zap_name_t *zn,
continue;

if (zn == NULL) {
zn = zap_name_alloc(zap, name, MT_NORMALIZE);
zn = zap_name_alloc_str(zap, name, MT_NORMALIZE);
allocdzn = B_TRUE;
}
if (zap_leaf_array_match(zeh->zeh_leaf, zn,
Expand Down
Loading