Skip to content

Commit

Permalink
ABD changes for vectorized RAIDZ
Browse files Browse the repository at this point in the history 
- userspace: aligned buffers. Minimum of 32B alignment is needed for AVX2. Kernel buffers are aligned 512B or more.

- add abd_get_offset_size() interface

- abd_iter_map(): fix calculation of iter_mapsize

- add abd_raidz_gen_iterate() and abd_raidz_rec_iterate()

Signed-off-by: Gvozden Neskovic <[email protected]>
  • Loading branch information
@ironMann
ironMann committed Aug 24, 2016
1 parent 7f53e14 commit 2d660f3
Show file tree
Hide file tree
Showing 3 changed files with 211 additions and 12 deletions.
12 changes: 12 additions & 0 deletions include/sys/abd.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ abd_t *abd_alloc_for_io_nosleep(size_t, boolean_t);
abd_t *abd_alloc_sametype(abd_t *, size_t);
void abd_free(abd_t *);
abd_t *abd_get_offset(abd_t *, size_t);
abd_t *abd_get_offset_size(abd_t *, size_t, size_t);
abd_t *abd_get_from_buf(void *, size_t);
void abd_put(abd_t *);

Expand Down Expand Up @@ -113,6 +114,17 @@ int abd_cmp(abd_t *, abd_t *);
int abd_cmp_buf_off(abd_t *, const void *, size_t, size_t);
void abd_zero_off(abd_t *, size_t, size_t);



void abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
ssize_t csize, ssize_t dsize, const unsigned parity,
void (*func_raidz_gen)(void **, const void *, size_t, size_t));
void abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
ssize_t tsize, const unsigned parity,
void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
const unsigned *mul),
const unsigned *mul);

/*
* Wrappers for calls with offsets of 0
*/
Expand Down
195 changes: 189 additions & 6 deletions module/zfs/abd.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,8 +228,8 @@ abd_fini(void)
struct page;
#define kpm_enable 1
#define abd_alloc_chunk() \
((struct page *)kmem_alloc(PAGESIZE, KM_SLEEP))
#define abd_free_chunk(chunk) kmem_free(chunk, PAGESIZE)
((struct page *) umem_alloc_aligned(PAGESIZE, 64, KM_SLEEP))
#define abd_free_chunk(chunk) umem_free(chunk, PAGESIZE)
#define abd_map_chunk(chunk) ((void *)chunk)
static void
abd_unmap_chunk(struct page *c)
Expand Down Expand Up @@ -481,8 +481,8 @@ abd_alloc_for_io_nosleep(size_t size, boolean_t is_metadata)
* buffer data with sabd. Use abd_put() to free. sabd must not be freed while
* any derived ABDs exist.
*/
abd_t *
abd_get_offset(abd_t *sabd, size_t off)
static inline abd_t *
abd_get_offset_impl(abd_t *sabd, size_t off, size_t size)
{
abd_t *abd;

Expand Down Expand Up @@ -524,14 +524,32 @@ abd_get_offset(abd_t *sabd, size_t off)
chunkcnt * sizeof (void *));
}

abd->abd_size = sabd->abd_size - off;
abd->abd_size = size;
abd->abd_parent = sabd;
refcount_create(&abd->abd_children);
(void) refcount_add_many(&sabd->abd_children, abd->abd_size, abd);

return (abd);
}

abd_t *
abd_get_offset(abd_t *sabd, size_t off)
{
size_t size = sabd->abd_size > off ? sabd->abd_size - off : 0;

VERIFY3U(size, >, 0);

return (abd_get_offset_impl(sabd, off, size));
}

abd_t *
abd_get_offset_size(abd_t *sabd, size_t off, size_t size)
{
ASSERT3U(off + size, <=, sabd->abd_size);

return (abd_get_offset_impl(sabd, off, size));
}

/*
* Allocate a linear ABD structure for buf. You must free this with abd_put()
* since the resulting ABD doesn't own its own buffer.
Expand Down Expand Up @@ -764,10 +782,14 @@ abd_iter_map(struct abd_iter *aiter)
} else {
size_t index = abd_iter_scatter_chunk_index(aiter);
offset = abd_iter_scatter_chunk_offset(aiter);
aiter->iter_mapsize = PAGESIZE - offset;

aiter->iter_mapsize = MIN(PAGESIZE - offset,
aiter->iter_abd->abd_size - aiter->iter_pos);

paddr = abd_map_chunk(
aiter->iter_abd->abd_u.abd_scatter.abd_chunks[index]);
}

aiter->iter_mapaddr = (char *)paddr + offset;
}

Expand Down Expand Up @@ -1007,6 +1029,167 @@ abd_cmp(abd_t *dabd, abd_t *sabd)
}


/*
* Iterate over code ABDs and a data ABD and call @func_raidz_gen.
*
* @cabds parity ABDs, must have equal size
* @dabd data ABD. Can be NULL (in this case @dsize = 0)
* @func_raidz_gen should be implemented so that its behaviour
* is the same when taking linear and when taking scatter
*/
void
abd_raidz_gen_iterate(abd_t **cabds, abd_t *dabd,
ssize_t csize, ssize_t dsize, const unsigned parity,
void (*func_raidz_gen)(void **, const void *, size_t, size_t))
{
int i;
ssize_t len, dlen;
struct abd_iter caiters[3];
struct abd_iter daiter;
void *caddrs[3];

ASSERT3U(parity, <=, 3);

for (i = 0; i < parity; i++)
abd_iter_init(&caiters[i], cabds[i]);

if (dabd)
abd_iter_init(&daiter, dabd);

ASSERT3S(dsize, >=, 0);

while (csize > 0) {
len = csize;

if (dabd && dsize > 0)
abd_iter_map(&daiter);

for (i = 0; i < parity; i++) {
abd_iter_map(&caiters[i]);
caddrs[i] = caiters[i].iter_mapaddr;
}

switch (parity) {
case 3:
len = MIN(caiters[2].iter_mapsize, len);
case 2:
len = MIN(caiters[1].iter_mapsize, len);
case 1:
len = MIN(caiters[0].iter_mapsize, len);
}

/* must be progressive */
ASSERT3S(len, >, 0);

if (dabd && dsize > 0) {
/* this needs precise iter.length */
len = MIN(daiter.iter_mapsize, len);
dlen = len;
} else
dlen = 0;

/* must be progressive */
ASSERT3S(len, >, 0);
/*
* The iterated function likely will not do well if each
* segment except the last one is not multiple of 512 (raidz).
*/
ASSERT3U(((uint64_t)len & 511ULL), ==, 0);

func_raidz_gen(caddrs, daiter.iter_mapaddr, len, dlen);

for (i = parity-1; i >= 0; i--) {
abd_iter_unmap(&caiters[i]);
abd_iter_advance(&caiters[i], len);
}

if (dabd && dsize > 0) {
abd_iter_unmap(&daiter);
abd_iter_advance(&daiter, dlen);
dsize -= dlen;
}

csize -= len;

ASSERT3S(dsize, >=, 0);
ASSERT3S(csize, >=, 0);
}
}

/*
* Iterate over code ABDs and data reconstruction target ABDs and call
* @func_raidz_rec. Function maps at most 6 pages atomically.
*
* @cabds parity ABDs, must have equal size
* @tabds rec target ABDs, at most 3
* @tsize size of data target columns
* @func_raidz_rec expects syndrome data in target columns. Function
* reconstructs data and overwrites target columns.
*/
void
abd_raidz_rec_iterate(abd_t **cabds, abd_t **tabds,
ssize_t tsize, const unsigned parity,
void (*func_raidz_rec)(void **t, const size_t tsize, void **c,
const unsigned *mul),
const unsigned *mul)
{
int i;
ssize_t len;
struct abd_iter citers[3];
struct abd_iter xiters[3];
void *caddrs[3], *xaddrs[3];

ASSERT3U(parity, <=, 3);

for (i = 0; i < parity; i++) {
abd_iter_init(&citers[i], cabds[i]);
abd_iter_init(&xiters[i], tabds[i]);
}

while (tsize > 0) {

for (i = 0; i < parity; i++) {
abd_iter_map(&citers[i]);
abd_iter_map(&xiters[i]);
caddrs[i] = citers[i].iter_mapaddr;
xaddrs[i] = xiters[i].iter_mapaddr;
}

len = tsize;
switch (parity) {
case 3:
len = MIN(xiters[2].iter_mapsize, len);
len = MIN(citers[2].iter_mapsize, len);
case 2:
len = MIN(xiters[1].iter_mapsize, len);
len = MIN(citers[1].iter_mapsize, len);
case 1:
len = MIN(xiters[0].iter_mapsize, len);
len = MIN(citers[0].iter_mapsize, len);
}
/* must be progressive */
ASSERT3S(len, >, 0);
/*
* The iterated function likely will not do well if each
* segment except the last one is not multiple of 512 (raidz).
*/
ASSERT3U(((uint64_t)len & 511ULL), ==, 0);

func_raidz_rec(xaddrs, len, caddrs, mul);

for (i = parity-1; i >= 0; i--) {
abd_iter_unmap(&xiters[i]);
abd_iter_unmap(&citers[i]);
abd_iter_advance(&xiters[i], len);
abd_iter_advance(&citers[i], len);
}

tsize -= len;
ASSERT3S(tsize, >=, 0);
}
}


#if defined(_KERNEL) && defined(HAVE_SPL)
/* Tunable Parameters */
module_param(zfs_abd_scatter_enabled, int, 0644);
Expand Down
16 changes: 10 additions & 6 deletions module/zfs/vdev_raidz.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,8 +243,9 @@ vdev_raidz_cksum_finish(zio_cksum_report_t *zcr, const void *good_data)
offset = 0;
for (x = rm->rm_firstdatacol; x < rm->rm_cols; x++) {
abd_put(rm->rm_col[x].rc_abd);
rm->rm_col[x].rc_abd = abd_get_offset(
rm->rm_abd_copy, offset);
rm->rm_col[x].rc_abd = abd_get_offset_size(
rm->rm_abd_copy, offset,
rm->rm_col[x].rc_size);
offset += rm->rm_col[x].rc_size;
}
}
Expand Down Expand Up @@ -310,7 +311,8 @@ vdev_raidz_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *arg)

for (offset = 0, c = rm->rm_firstdatacol; c < rm->rm_cols; c++) {
raidz_col_t *col = &rm->rm_col[c];
abd_t *tmp = abd_get_offset(rm->rm_abd_copy, offset);
abd_t *tmp = abd_get_offset_size(rm->rm_abd_copy, offset,
col->rc_size);

abd_copy(tmp, col->rc_abd, col->rc_size);
abd_put(col->rc_abd);
Expand Down Expand Up @@ -432,13 +434,15 @@ vdev_raidz_map_alloc(zio_t *zio, uint64_t unit_shift, uint64_t dcols,

for (c = 0; c < rm->rm_firstdatacol; c++)
rm->rm_col[c].rc_abd =
abd_alloc_linear(rm->rm_col[c].rc_size, B_TRUE);
abd_alloc_linear(rm->rm_col[c].rc_size, B_FALSE);

rm->rm_col[c].rc_abd = abd_get_offset(zio->io_abd, 0);
rm->rm_col[c].rc_abd = abd_get_offset_size(zio->io_abd, 0,
rm->rm_col[c].rc_size);
off = rm->rm_col[c].rc_size;

for (c = c + 1; c < acols; c++) {
rm->rm_col[c].rc_abd = abd_get_offset(zio->io_abd, off);
rm->rm_col[c].rc_abd = abd_get_offset_size(zio->io_abd, off,
rm->rm_col[c].rc_size);
off += rm->rm_col[c].rc_size;
}

Expand Down

0 comments on commit 2d660f3

Please sign in to comment.