Skip to content

Commit

Permalink
Illumos 5746 more checksumming in zfs send
Browse files Browse the repository at this point in the history 
Reviewed by: Christopher Siden <[email protected]>
Reviewed by: George Wilson <[email protected]>
Reviewed by: Bayard Bell <[email protected]>
Approved by: Albert Lee <[email protected]>

References:
illumos/illumos-gate@98110f0
https://www.illumos.org/issues/5746

openzfs#905
(Add "zstreamdump" integrity check program openzfs#905)
openzfs@b79fc3f
(Add zstreamdump(8) command to examine ZFS send streams.)

Porting notes:
Some of those changes were somewhat hard to track due to:
openzfs@2024041 Remove superfluous statement

openzfs@044baf0 Use taskq for dump_bytes()

openzfs@88904bb Illumos 5162 - zfs recv should use loaned arc buffer to avoid copy

kmem_alloc calls were changed to vmem_alloc
in accordance with openzfs@77aef6f (Use vmem_alloc() for nvlists) and the kmem-rework

account for ISO C90 warnings (-Werror=declaration-after-statement)
account for error: format '%llx' expects argument of type 'long long unsigned int', but argument X has type 'uint64_t' [-Werror=format=]

arc_buf_t *abuf;
dmu_buf_t *bonus;
zio_cksum_t cksum_orig;
zio_cksum_t *cksump;

Fix whitespace

change
(void) fprintf(stderr, "ERROR; failed to allocate %u bytes\n",
(unsigned)size);
to
(void) fprintf(stderr, "ERROR; failed to allocate %zu bytes\n",
size);

and to account for long unsigned int & match upstream

Ported-by: kernelOfTruth [email protected]
  • Loading branch information
@ahrens @kernelOfTruth
ahrens authored and kernelOfTruth committed Dec 10, 2015
1 parent 24ef51f commit 258d7d9
Show file tree
Hide file tree
Showing 6 changed files with 444 additions and 306 deletions.
60 changes: 50 additions & 10 deletions cmd/zstreamdump/zstreamdump.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
*/

/*
* Copyright (c) 2013 by Delphix. All rights reserved.
* Copyright (c) 2013, 2014 by Delphix. All rights reserved.
*/

#include <ctype.h>
Expand All @@ -36,6 +36,7 @@
#include <stdlib.h>
#include <strings.h>
#include <unistd.h>
#include <stddef.h>

#include <sys/dmu.h>
#include <sys/zfs_ioctl.h>
Expand Down Expand Up @@ -73,8 +74,8 @@ safe_malloc(size_t size)
{
void *rv = malloc(size);
if (rv == NULL) {
(void) fprintf(stderr, "ERROR; failed to allocate %u bytes\n",
(unsigned)size);
(void) fprintf(stderr, "ERROR; failed to allocate %zu bytes\n",
size);
abort();
}
return (rv);
Expand All @@ -85,7 +86,6 @@ safe_malloc(size_t size)
*
* Read while computing incremental checksum
*/

static size_t
ssread(void *buf, size_t len, zio_cksum_t *cksum)
{
Expand All @@ -94,7 +94,7 @@ ssread(void *buf, size_t len, zio_cksum_t *cksum)
if ((outlen = fread(buf, len, 1, send_stream)) == 0)
return (0);

if (do_cksum && cksum) {
if (do_cksum) {
if (do_byteswap)
fletcher_4_incremental_byteswap(buf, len, cksum);
else
Expand All @@ -104,6 +104,34 @@ ssread(void *buf, size_t len, zio_cksum_t *cksum)
return (outlen);
}

static size_t
read_hdr(dmu_replay_record_t *drr, zio_cksum_t *cksum)
{
ASSERT3U(offsetof(dmu_replay_record_t, drr_u.drr_checksum.drr_checksum),
==, sizeof (dmu_replay_record_t) - sizeof (zio_cksum_t));
size_t r = ssread(drr, sizeof (*drr) - sizeof (zio_cksum_t), cksum);
if (r == 0)
return (0);
zio_cksum_t saved_cksum = *cksum;
r = ssread(&drr->drr_u.drr_checksum.drr_checksum,
sizeof (zio_cksum_t), cksum);
if (r == 0)
return (0);
if (!ZIO_CHECKSUM_IS_ZERO(&drr->drr_u.drr_checksum.drr_checksum) &&
!ZIO_CHECKSUM_EQUAL(saved_cksum,
drr->drr_u.drr_checksum.drr_checksum)) {
fprintf(stderr, "invalid checksum\n");
(void) printf("Incorrect checksum in record header.\n");
(void) printf("Expected checksum = %llx/%llx/%llx/%llx\n",
(long long unsigned int)saved_cksum.zc_word[0],
(long long unsigned int)saved_cksum.zc_word[1],
(long long unsigned int)saved_cksum.zc_word[2],
(long long unsigned int)saved_cksum.zc_word[3]);
exit(1);
}
return (sizeof (*drr));
}

/*
* Print part of a block in ASCII characters
*/
Expand Down Expand Up @@ -185,8 +213,10 @@ main(int argc, char *argv[])
struct drr_free *drrf = &thedrr.drr_u.drr_free;
struct drr_spill *drrs = &thedrr.drr_u.drr_spill;
struct drr_write_embedded *drrwe = &thedrr.drr_u.drr_write_embedded;
struct drr_checksum *drrc = &thedrr.drr_u.drr_checksum;
char c;
boolean_t verbose = B_FALSE;
boolean_t very_verbose = B_FALSE;
boolean_t first = B_TRUE;
/*
* dump flag controls whether the contents of any modified data blocks
Expand All @@ -204,11 +234,14 @@ main(int argc, char *argv[])
do_cksum = B_FALSE;
break;
case 'v':
if (verbose)
very_verbose = B_TRUE;
verbose = B_TRUE;
break;
case 'd':
dump = B_TRUE;
verbose = B_TRUE;
very_verbose = B_TRUE;
break;
case ':':
(void) fprintf(stderr,
Expand All @@ -231,7 +264,7 @@ main(int argc, char *argv[])
}

send_stream = stdin;
while (ssread(drr, sizeof (dmu_replay_record_t), &zc)) {
while (read_hdr(drr, &zc)) {

/*
* If this is the first DMU record being processed, check for
Expand Down Expand Up @@ -437,7 +470,7 @@ main(int argc, char *argv[])
if (verbose) {
(void) printf("WRITE object = %llu type = %u "
"checksum type = %u\n"
"offset = %llu length = %llu "
" offset = %llu length = %llu "
"props = %llx\n",
(u_longlong_t)drrw->drr_object,
drrw->drr_type,
Expand Down Expand Up @@ -481,9 +514,9 @@ main(int argc, char *argv[])
if (verbose) {
(void) printf("WRITE_BYREF object = %llu "
"checksum type = %u props = %llx\n"
"offset = %llu length = %llu\n"
" offset = %llu length = %llu\n"
"toguid = %llx refguid = %llx\n"
"refobject = %llu refoffset = %llu\n",
" refobject = %llu refoffset = %llu\n",
(u_longlong_t)drrwbr->drr_object,
drrwbr->drr_checksumtype,
(u_longlong_t)drrwbr->drr_key.ddk_prop,
Expand Down Expand Up @@ -544,7 +577,7 @@ main(int argc, char *argv[])
if (verbose) {
(void) printf("WRITE_EMBEDDED object = %llu "
"offset = %llu length = %llu\n"
"toguid = %llx comp = %u etype = %u "
" toguid = %llx comp = %u etype = %u "
"lsize = %u psize = %u\n",
(u_longlong_t)drrwe->drr_object,
(u_longlong_t)drrwe->drr_offset,
Expand All @@ -562,6 +595,13 @@ main(int argc, char *argv[])
/* should never be reached */
exit(1);
}
if (drr->drr_type != DRR_BEGIN && very_verbose) {
(void) printf(" checksum = %llx/%llx/%llx/%llx\n",
(longlong_t)drrc->drr_checksum.zc_word[0],
(longlong_t)drrc->drr_checksum.zc_word[1],
(longlong_t)drrc->drr_checksum.zc_word[2],
(longlong_t)drrc->drr_checksum.zc_word[3]);
}
pcksum = zc;
}
free(buf);
Expand Down
13 changes: 13 additions & 0 deletions include/sys/spa.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -446,6 +446,19 @@ _NOTE(CONSTCOND) } while (0)
((zc1).zc_word[2] - (zc2).zc_word[2]) | \
((zc1).zc_word[3] - (zc2).zc_word[3])))

#define ZIO_CHECKSUM_IS_ZERO(zc) \
(0 == ((zc)->zc_word[0] | (zc)->zc_word[1] | \
(zc)->zc_word[2] | (zc)->zc_word[3]))

#define ZIO_CHECKSUM_BSWAP(zcp) \
{ \
(zcp)->zc_word[0] = BSWAP_64((zcp)->zc_word[0]); \
(zcp)->zc_word[1] = BSWAP_64((zcp)->zc_word[1]); \
(zcp)->zc_word[2] = BSWAP_64((zcp)->zc_word[2]); \
(zcp)->zc_word[3] = BSWAP_64((zcp)->zc_word[3]); \
}


#define DVA_IS_VALID(dva) (DVA_GET_ASIZE(dva) != 0)

#define ZIO_SET_CHECKSUM(zcp, w0, w1, w2, w3) \
Expand Down
18 changes: 17 additions & 1 deletion include/sys/zfs_ioctl.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013 by Delphix. All rights reserved.
* Copyright (c) 2012, 2014 by Delphix. All rights reserved.
*/

#ifndef _SYS_ZFS_IOCTL_H
Expand Down Expand Up @@ -237,6 +237,22 @@ typedef struct dmu_replay_record {
uint32_t drr_psize; /* compr. (real) size of payload */
/* (possibly compressed) content follows */
} drr_write_embedded;

/*
* Nore: drr_checksum is overlaid with all record types
* except DRR_BEGIN. Therefore its (non-pad) members
* must not overlap with members from the other structs.
* We accomplish this by putting its members at the very
* end of the struct.
*/
struct drr_checksum {
uint64_t drr_pad[34];
/*
* fletcher-4 checksum of everything preceding the
* checksum.
*/
zio_cksum_t drr_checksum;
} drr_checksum;
} drr_u;
} dmu_replay_record_t;

Expand Down
7 changes: 4 additions & 3 deletions include/sys/zio_checksum.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
*/
/*
* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2014 by Delphix. All rights reserved.
*/

#ifndef _SYS_ZIO_CHECKSUM_H
Expand All @@ -34,13 +35,13 @@ extern "C" {
/*
* Signature for checksum functions.
*/
typedef void zio_checksum_t(const void *data, uint64_t size, zio_cksum_t *zcp);
typedef void zio_checksum_func_t(const void *, uint64_t, zio_cksum_t *);

/*
* Information about each checksum function.
*/
typedef const struct zio_checksum_info {
zio_checksum_t *ci_func[2]; /* checksum function for each byteorder */
zio_checksum_func_t *ci_func[2]; /* checksum function per byteorder */
int ci_correctable; /* number of correctable bits */
int ci_eck; /* uses zio embedded checksum? */
int ci_dedup; /* strong enough for dedup? */
Expand All @@ -61,7 +62,7 @@ extern zio_checksum_info_t zio_checksum_table[ZIO_CHECKSUM_FUNCTIONS];
/*
* Checksum routines.
*/
extern zio_checksum_t zio_checksum_SHA256;
extern zio_checksum_func_t zio_checksum_SHA256;

extern void zio_checksum_compute(zio_t *zio, enum zio_checksum checksum,
void *data, uint64_t size);
Expand Down
Loading

0 comments on commit 258d7d9

Please sign in to comment.