Skip to content

Commit

Permalink
Merge branch 'zil-performance'
Browse files Browse the repository at this point in the history 
This brnach brings some ZIL performance optimizations, with
significant increases in synchronous write performance for
some workloads and pool configurations.

See the individual commit messages for details.

Signed-off-by: Brian Behlendorf <[email protected]>
Closes #1013
  • Loading branch information
@behlendorf
behlendorf committed Oct 17, 2012
2 parents 82f4673 + 5d7a86d commit 658a014
Show file tree
Hide file tree
Showing 14 changed files with 434 additions and 27 deletions.
3 changes: 3 additions & 0 deletions include/sys/metaslab.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,12 +50,15 @@ extern void metaslab_sync_reassess(metaslab_group_t *mg);
#define METASLAB_GANG_HEADER 0x2
#define METASLAB_GANG_CHILD 0x4
#define METASLAB_GANG_AVOID 0x8
#define METASLAB_FASTWRITE 0x10

extern int metaslab_alloc(spa_t *spa, metaslab_class_t *mc, uint64_t psize,
blkptr_t *bp, int ncopies, uint64_t txg, blkptr_t *hintbp, int flags);
extern void metaslab_free(spa_t *spa, const blkptr_t *bp, uint64_t txg,
boolean_t now);
extern int metaslab_claim(spa_t *spa, const blkptr_t *bp, uint64_t txg);
extern void metaslab_fastwrite_mark(spa_t *spa, const blkptr_t *bp);
extern void metaslab_fastwrite_unmark(spa_t *spa, const blkptr_t *bp);

extern metaslab_class_t *metaslab_class_create(spa_t *spa,
space_map_ops_t *ops);
Expand Down
1 change: 1 addition & 0 deletions include/sys/metaslab_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ struct metaslab_class {
uint64_t mc_deferred; /* total deferred frees */
uint64_t mc_space; /* total space (alloc + free) */
uint64_t mc_dspace; /* total deflated space */
kmutex_t mc_fastwrite_lock;
};

struct metaslab_group {
Expand Down
1 change: 1 addition & 0 deletions include/sys/vdev_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ struct vdev {
uint64_t vdev_ms_count; /* number of metaslabs */
metaslab_group_t *vdev_mg; /* metaslab group */
metaslab_t **vdev_ms; /* metaslab array */
uint64_t vdev_pending_fastwrite; /* allocated fastwrites */
txg_list_t vdev_ms_list; /* per-txg dirty metaslab lists */
txg_list_t vdev_dtl_list; /* per-txg dirty DTL lists */
txg_node_t vdev_txg_node; /* per-txg dirty vdev linkage */
Expand Down
1 change: 1 addition & 0 deletions include/sys/zil_impl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,7 @@ extern "C" {
typedef struct lwb {
zilog_t *lwb_zilog; /* back pointer to log struct */
blkptr_t lwb_blk; /* on disk address of this log blk */
boolean_t lwb_fastwrite; /* is blk marked for fastwrite? */
int lwb_nused; /* # used bytes in buffer */
int lwb_sz; /* size of block and buffer */
char *lwb_buf; /* log write buffer */
Expand Down
5 changes: 3 additions & 2 deletions include/sys/zio.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -193,7 +193,8 @@ enum zio_flag {
ZIO_FLAG_RAW = 1 << 21,
ZIO_FLAG_GANG_CHILD = 1 << 22,
ZIO_FLAG_DDT_CHILD = 1 << 23,
ZIO_FLAG_GODFATHER = 1 << 24
ZIO_FLAG_GODFATHER = 1 << 24,
ZIO_FLAG_FASTWRITE = 1 << 25
};

#define ZIO_FLAG_MUSTSUCCEED 0
Expand Down Expand Up @@ -475,7 +476,7 @@ extern zio_t *zio_free_sync(zio_t *pio, spa_t *spa, uint64_t txg,
const blkptr_t *bp, enum zio_flag flags);

extern int zio_alloc_zil(spa_t *spa, uint64_t txg, blkptr_t *new_bp,
blkptr_t *old_bp, uint64_t size, boolean_t use_slog);
uint64_t size, boolean_t use_slog);
extern void zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp);
extern void zio_flush(zio_t *zio, vdev_t *vd);
extern void zio_shrink(zio_t *zio, uint64_t size);
Expand Down
56 changes: 56 additions & 0 deletions lib/libspl/asm-generic/atomic.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,6 +103,31 @@ void atomic_add_ptr(volatile void *target, ssize_t bits)
}


#define ATOMIC_SUB(name, type1, type2) \
void atomic_sub_##name(volatile type1 *target, type2 bits) \
{ \
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
*target -= bits; \
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
}

ATOMIC_SUB(8, uint8_t, int8_t)
ATOMIC_SUB(char, uchar_t, signed char)
ATOMIC_SUB(16, uint16_t, int16_t)
ATOMIC_SUB(short, ushort_t, short)
ATOMIC_SUB(32, uint32_t, int32_t)
ATOMIC_SUB(int, uint_t, int)
ATOMIC_SUB(long, ulong_t, long)
ATOMIC_SUB(64, uint64_t, int64_t)

void atomic_sub_ptr(volatile void *target, ssize_t bits)
{
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
*(caddr_t *)target -= bits;
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);
}


#define ATOMIC_OR(name, type) \
void atomic_or_##name(volatile type *target, type bits) \
{ \
Expand Down Expand Up @@ -216,6 +241,37 @@ void *atomic_add_ptr_nv(volatile void *target, ssize_t bits)
}


#define ATOMIC_SUB_NV(name, type1, type2) \
type1 atomic_sub_##name##_nv(volatile type1 *target, type2 bits)\
{ \
type1 rc; \
VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0); \
rc = (*target -= bits); \
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0); \
return rc; \
}

ATOMIC_SUB_NV(8, uint8_t, int8_t)
ATOMIC_SUB_NV(char, uchar_t, signed char)
ATOMIC_SUB_NV(16, uint16_t, int16_t)
ATOMIC_SUB_NV(short, ushort_t, short)
ATOMIC_SUB_NV(32, uint32_t, int32_t)
ATOMIC_SUB_NV(int, uint_t, int)
ATOMIC_SUB_NV(long, ulong_t, long)
ATOMIC_SUB_NV(64, uint64_t, int64_t)

void *atomic_sub_ptr_nv(volatile void *target, ssize_t bits)
{
void *ptr;

VERIFY3S(pthread_mutex_lock(&atomic_lock), ==, 0);
ptr = (*(caddr_t *)target -= bits);
VERIFY3S(pthread_mutex_unlock(&atomic_lock), ==, 0);

return ptr;
}


#define ATOMIC_OR_NV(name, type) \
type atomic_or_##name##_nv(volatile type *target, type bits) \
{ \
Expand Down
106 changes: 106 additions & 0 deletions lib/libspl/asm-i386/atomic.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,40 @@
SET_SIZE(atomic_add_int)
SET_SIZE(atomic_add_32)

ENTRY(atomic_sub_8)
ALTENTRY(atomic_sub_char)
movl 4(%esp), %eax
movl 8(%esp), %ecx
lock
subb %cl, (%eax)
ret
SET_SIZE(atomic_sub_char)
SET_SIZE(atomic_sub_8)

ENTRY(atomic_sub_16)
ALTENTRY(atomic_sub_short)
movl 4(%esp), %eax
movl 8(%esp), %ecx
lock
subw %cx, (%eax)
ret
SET_SIZE(atomic_sub_short)
SET_SIZE(atomic_sub_16)

ENTRY(atomic_sub_32)
ALTENTRY(atomic_sub_int)
ALTENTRY(atomic_sub_ptr)
ALTENTRY(atomic_sub_long)
movl 4(%esp), %eax
movl 8(%esp), %ecx
lock
subl %ecx, (%eax)
ret
SET_SIZE(atomic_sub_long)
SET_SIZE(atomic_sub_ptr)
SET_SIZE(atomic_sub_int)
SET_SIZE(atomic_sub_32)

ENTRY(atomic_or_8)
ALTENTRY(atomic_or_uchar)
movl 4(%esp), %eax
Expand Down Expand Up @@ -384,6 +418,55 @@
SET_SIZE(atomic_add_int_nv)
SET_SIZE(atomic_add_32_nv)

ENTRY(atomic_sub_8_nv)
ALTENTRY(atomic_sub_char_nv)
movl 4(%esp), %edx
movb (%edx), %al
1:
movl 8(%esp), %ecx
subb %al, %cl
lock
cmpxchgb %cl, (%edx)
jne 1b
movzbl %cl, %eax
ret
SET_SIZE(atomic_sub_char_nv)
SET_SIZE(atomic_sub_8_nv)

ENTRY(atomic_sub_16_nv)
ALTENTRY(atomic_sub_short_nv)
movl 4(%esp), %edx
movw (%edx), %ax
1:
movl 8(%esp), %ecx
subw %ax, %cx
lock
cmpxchgw %cx, (%edx)
jne 1b
movzwl %cx, %eax
ret
SET_SIZE(atomic_sub_short_nv)
SET_SIZE(atomic_sub_16_nv)

ENTRY(atomic_sub_32_nv)
ALTENTRY(atomic_sub_int_nv)
ALTENTRY(atomic_sub_ptr_nv)
ALTENTRY(atomic_sub_long_nv)
movl 4(%esp), %edx
movl (%edx), %eax
1:
movl 8(%esp), %ecx
subl %eax, %ecx
lock
cmpxchgl %ecx, (%edx)
jne 1b
movl %ecx, %eax
ret
SET_SIZE(atomic_sub_long_nv)
SET_SIZE(atomic_sub_ptr_nv)
SET_SIZE(atomic_sub_int_nv)
SET_SIZE(atomic_sub_32_nv)

/*
* NOTE: If atomic_add_64 and atomic_add_64_nv are ever
* separated, it is important to edit the libc i386 platform
Expand Down Expand Up @@ -413,6 +496,29 @@
SET_SIZE(atomic_add_64_nv)
SET_SIZE(atomic_add_64)

ENTRY(atomic_sub_64)
ALTENTRY(atomic_sub_64_nv)
pushl %edi
pushl %ebx
movl 12(%esp), %edi
movl (%edi), %eax
movl 4(%edi), %edx
1:
movl 16(%esp), %ebx
movl 20(%esp), %ecx
subl %eax, %ebx
adcl %edx, %ecx
lock
cmpxchg8b (%edi)
jne 1b
movl %ebx, %eax
movl %ecx, %edx
popl %ebx
popl %edi
ret
SET_SIZE(atomic_sub_64_nv)
SET_SIZE(atomic_sub_64)

ENTRY(atomic_or_8_nv)
ALTENTRY(atomic_or_uchar_nv)
movl 4(%esp), %edx
Expand Down
92 changes: 92 additions & 0 deletions lib/libspl/asm-x86_64/atomic.S
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,6 +232,40 @@
SET_SIZE(atomic_add_ptr)
SET_SIZE(atomic_add_64)

ENTRY(atomic_sub_8)
ALTENTRY(atomic_sub_char)
lock
subb %sil, (%rdi)
ret
SET_SIZE(atomic_sub_char)
SET_SIZE(atomic_sub_8)

ENTRY(atomic_sub_16)
ALTENTRY(atomic_sub_short)
lock
subw %si, (%rdi)
ret
SET_SIZE(atomic_sub_short)
SET_SIZE(atomic_sub_16)

ENTRY(atomic_sub_32)
ALTENTRY(atomic_sub_int)
lock
subl %esi, (%rdi)
ret
SET_SIZE(atomic_sub_int)
SET_SIZE(atomic_sub_32)

ENTRY(atomic_sub_64)
ALTENTRY(atomic_sub_ptr)
ALTENTRY(atomic_sub_long)
lock
subq %rsi, (%rdi)
ret
SET_SIZE(atomic_sub_long)
SET_SIZE(atomic_sub_ptr)
SET_SIZE(atomic_sub_64)

ENTRY(atomic_or_8)
ALTENTRY(atomic_or_uchar)
lock
Expand Down Expand Up @@ -354,6 +388,64 @@
SET_SIZE(atomic_add_ptr_nv)
SET_SIZE(atomic_add_64_nv)

ENTRY(atomic_sub_8_nv)
ALTENTRY(atomic_sub_char_nv)
movb (%rdi), %al
1:
movb %sil, %cl
subb %al, %cl
lock
cmpxchgb %cl, (%rdi)
jne 1b
movzbl %cl, %eax
ret
SET_SIZE(atomic_sub_char_nv)
SET_SIZE(atomic_sub_8_nv)

ENTRY(atomic_sub_16_nv)
ALTENTRY(atomic_sub_short_nv)
movw (%rdi), %ax
1:
movw %si, %cx
subw %ax, %cx
lock
cmpxchgw %cx, (%rdi)
jne 1b
movzwl %cx, %eax
ret
SET_SIZE(atomic_sub_short_nv)
SET_SIZE(atomic_sub_16_nv)

ENTRY(atomic_sub_32_nv)
ALTENTRY(atomic_sub_int_nv)
movl (%rdi), %eax
1:
movl %esi, %ecx
subl %eax, %ecx
lock
cmpxchgl %ecx, (%rdi)
jne 1b
movl %ecx, %eax
ret
SET_SIZE(atomic_sub_int_nv)
SET_SIZE(atomic_sub_32_nv)

ENTRY(atomic_sub_64_nv)
ALTENTRY(atomic_sub_ptr_nv)
ALTENTRY(atomic_sub_long_nv)
movq (%rdi), %rax
1:
movq %rsi, %rcx
subq %rax, %rcx
lock
cmpxchgq %rcx, (%rdi)
jne 1b
movq %rcx, %rax
ret
SET_SIZE(atomic_sub_long_nv)
SET_SIZE(atomic_sub_ptr_nv)
SET_SIZE(atomic_sub_64_nv)

ENTRY(atomic_and_8_nv)
ALTENTRY(atomic_and_uchar_nv)
movb (%rdi), %al
Expand Down
Loading

0 comments on commit 658a014

Please sign in to comment.