Skip to content

Commit

Permalink
docs: update benchmark
Browse files Browse the repository at this point in the history 
More useful comparison ratios and adds brotli
  • Loading branch information
@william-silversmith
william-silversmith committed Jan 12, 2024
1 parent fb99644 commit 64af0a1
Show file tree
Hide file tree
Showing 2 changed files with 162 additions and 156 deletions.
232 changes: 123 additions & 109 deletions benchmarks/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,123 +4,137 @@ Note that all numbers provided are for the "flat" label layout. The pin layout i
shape: (256, 256, 64)
PINKY40 CUTOUTS (connectomics.npy)

ckl: 28348
cpso: 67864 (41.77%)
raw: 16777216 (0.17%)
ckl.gz: 19228
cpso.gz 26256 (73.23%)
raw:.gz 71158 (27.02%)


ckl: 14516
cpso: 28496 (50.94%)
raw: 16777216 (0.09%)
ckl.gz: 7727
cpso.gz 8963 (86.21%)
raw:.gz 28636 (26.98%)


ckl: 24740
cpso: 56894 (43.48%)
raw: 16777216 (0.15%)
ckl.gz: 15809
cpso.gz 21184 (74.63%)
raw:.gz 59809 (26.43%)

ckl: 94049 (0.56%)
ckl(pin): 85699 (0.51%)
ckl(mkv): 76198 (0.45%)
cpso: 258208 (1.54%)
raw: 16777216 (100.00%)

ckl.gz: 71828 (27.47%)
ckl.gz(pin): 68383 (26.15%)
ckl.gz(mkv): 66263 (25.34%)
cpso.gz: 110145 (42.12%)
raw.gz: 261496 (100.00%)

ckl.br: 66497 (41.54%)
ckl.br(pin): 63759 (39.83%)
ckl.br(mkv): 63886 (39.91%)
cpso.br: 92544 (57.81%)
raw.br: 160076 (100.00%)
BINARY IMAGE (connectomics.npy, label 67699431)

ckl: 43401 (0.03%)
ckl(pin): 41996 (0.03%)
ckl(mkv): 34886 (0.03%)
cpso: 87480 (0.07%)
raw: 134217728 (100.00%)

ckl.gz: 24070 (11.62%)
ckl.gz(pin): 23862 (11.52%)
ckl.gz(mkv): 22577 (10.90%)
cpso.gz: 45130 (21.78%)
raw.gz: 207188 (100.00%)

ckl.br: 21552 (29.25%)
ckl.br(pin): 21434 (29.09%)
ckl.br(mkv): 21094 (28.63%)
cpso.br: 39110 (53.09%)
raw.br: 73672 (100.00%)
WATERSHED CUTOUTS (ws.npy)

ckl: 537617
cpso: 2515670 (21.37%)
raw: 33554432 (1.60%)
ckl.gz: 400151
cpso.gz 642979 (62.23%)
raw:.gz 970262 (41.24%)

ckl: 553454 (1.65%)
ckl(pin): 543172 (1.62%)
ckl(mkv): 492454 (1.47%)
cpso: 2546412 (7.59%)
raw: 33554432 (100.00%)

ckl.gz: 409447 (40.89%)
ckl.gz(pin): 403366 (40.28%)
ckl.gz(mkv): 396406 (39.58%)
cpso.gz: 657185 (65.62%)
raw.gz: 1001437 (100.00%)

ckl.br: 370373 (68.99%)
ckl.br(pin): 370793 (69.07%)
ckl.br(mkv): 369504 (68.82%)
cpso.br: 543861 (101.30%)
raw.br: 536875 (100.00%)
EMPTY

ckl: 561911
cpso: 2678712 (20.98%)
raw: 33554432 (1.67%)
ckl.gz: 421943
cpso.gz 680361 (62.02%)
raw:.gz 1012515 (41.67%)

ckl: 993 (0.01%)
ckl(pin): 930 (0.01%)
ckl(mkv): 993 (0.01%)
cpso: 824 (0.00%)
raw: 16777216 (100.00%)

ckl.gz: 44 (0.27%)
ckl.gz(pin): 42 (0.26%)
ckl.gz(mkv): 44 (0.27%)
cpso.gz: 43 (0.26%)
raw.gz: 16316 (100.00%)

ckl.br: 40 (148.15%)
ckl.br(pin): 40 (148.15%)
ckl.br(mkv): 40 (148.15%)
cpso.br: 46 (170.37%)
raw.br: 27 (100.00%)
SOLID ONES

ckl: 568996
cpso: 2670264 (21.31%)
raw: 33554432 (1.70%)
ckl.gz: 428341
cpso.gz 694560 (61.67%)
raw:.gz 1026001 (41.75%)

ckl: 993 (0.01%)
ckl(pin): 930 (0.01%)
ckl(mkv): 993 (0.01%)
cpso: 824 (0.00%)
raw: 16777216 (100.00%)

ckl.gz: 46 (0.28%)
ckl.gz(pin): 44 (0.27%)
ckl.gz(mkv): 46 (0.28%)
cpso.gz: 47 (0.29%)
raw.gz: 16322 (100.00%)

ckl.br: 41 (141.38%)
ckl.br(pin): 39 (134.48%)
ckl.br(mkv): 41 (141.38%)
cpso.br: 54 (186.21%)
raw.br: 29 (100.00%)
RANDOM NOISE [0,2000) uint32

ckl: 8405450
cpso: 17301994 (48.58%)
raw: 16777216 (50.10%)
ckl.gz: 6791314
cpso.gz 8024443 (84.63%)
raw:.gz 8023562 (84.64%)


ckl: 8405384
cpso: 17301996 (48.58%)
raw: 16777216 (50.10%)
ckl.gz: 6791995
cpso.gz 8024532 (84.64%)
raw:.gz 8023995 (84.65%)


ckl: 8405714
cpso: 17301984 (48.58%)
raw: 16777216 (50.10%)
ckl.gz: 6791443
cpso.gz 8024041 (84.64%)
raw:.gz 8023287 (84.65%)

ckl: 8412099 (50.14%)
ckl(pin): 8412099 (50.14%)
ckl(mkv): 8411377 (50.14%)
cpso: 17302012 (103.13%)
raw: 16777216 (100.00%)

ckl.gz: 6791243 (84.64%)
ckl.gz(pin): 6791243 (84.64%)
ckl.gz(mkv): 6791246 (84.64%)
cpso.gz: 8024331 (100.01%)
raw.gz: 8023874 (100.00%)

ckl.br: 5830287 (90.63%)
ckl.br(pin): 5830287 (90.63%)
ckl.br(mkv): 5829830 (90.62%)
cpso.br: 6571752 (102.16%)
raw.br: 6433114 (100.00%)
BINARY NOISE [0,1] uint8 (pathological case)

ckl: 1842719
cpso: 2899167 (63.56%)
raw: 4194304 (43.93%)
ckl.gz: 1193828
cpso.gz 1098040 (108.72%)
raw:.gz 667063 (178.97%)


ckl: 1843897
cpso: 2900235 (63.58%)
raw: 4194304 (43.96%)
ckl.gz: 1194222
cpso.gz 1098982 (108.67%)
raw:.gz 667228 (178.98%)


ckl: 1843267
cpso: 2897477 (63.62%)
raw: 4194304 (43.95%)
ckl.gz: 1195040
cpso.gz 1097542 (108.88%)
raw:.gz 667036 (179.16%)

EMPTY

ckl: 608
cpso: 824 (73.79%)
raw: 16777216 (0.00%)
ckl.gz: 35
cpso.gz 43 (81.40%)
raw:.gz 16316 (0.21%)

SOLID ONES

ckl: 608
cpso: 824 (73.79%)
raw: 16777216 (0.00%)
ckl.gz: 37
cpso.gz 47 (78.72%)
raw:.gz 16322 (0.23%)


ckl: 284350 (43.39%)
ckl(pin): 284350 (43.39%)
ckl(mkv): 270090 (41.21%)
cpso: 468337 (71.46%)
raw: 655360 (100.00%)

ckl.gz: 181347 (173.94%)
ckl.gz(pin): 181347 (173.94%)
ckl.gz(mkv): 181795 (174.37%)
cpso.gz: 186901 (179.27%)
raw.gz: 104259 (100.00%)

ckl.br: 170516 (208.09%)
ckl.br(pin): 170516 (208.09%)
ckl.br(mkv): 172139 (210.07%)
cpso.br: 162174 (197.91%)
raw.br: 81942 (100.00%)

## Performance Benchmark

Expand Down
86 changes: 39 additions & 47 deletions benchmarks/size.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import numpy as np
import zlib
import brotli

import time

Expand All @@ -16,47 +17,39 @@ def run_sample(labels, shape, N):
run(cutout)

def run(labels):
s = time.time()
ckl_binary = crackle.compress(labels, allow_pins=False)
compress_time = time.time() - s

s = time.time()
mkv_binary = crackle.compress(labels, allow_pins=False, markov_model_order=5)
mkv_compress_time = time.time() - s

s = time.time()
pin_binary = crackle.compress(labels, allow_pins=True)
pin_compress_time = time.time() - s

s = time.time()
cpso_binary = compresso.compress(labels)
cpso_time = time.time() - s

raw_binary = labels.tobytes('F')

s = time.time()
cckl_binary = zlib.compress(ckl_binary)
compress_time_gz = compress_time + (time.time() - s)

ccpso_binary = zlib.compress(cpso_binary)
craw_binary = zlib.compress(raw_binary)
cpin_binary = zlib.compress(pin_binary)
cmkv_binary = zlib.compress(mkv_binary)
ckl_binary = crackle.compress(labels, allow_pins=False)
mkv_binary = crackle.compress(labels, allow_pins=False, markov_model_order=5)
pin_binary = crackle.compress(labels, allow_pins=True)
cpso_binary = compresso.compress(labels)
raw_binary = labels.tobytes('F')

for method in ['','gz','br']:


if method == '':
fn = lambda x: x
ext = ''
elif method == 'gz':
fn = zlib.compress
ext = '.gz'
elif method == 'br':
fn = brotli.compress
ext = '.br'

cckl_binary = fn(ckl_binary)
ccpso_binary = fn(cpso_binary)
craw_binary = fn(raw_binary)
cpin_binary = fn(pin_binary)
cmkv_binary = fn(mkv_binary)

print(f"""
ckl: {len(ckl_binary): 9}
ckl(pin): {len(pin_binary): 9} ({len(pin_binary)/len(ckl_binary)*100:.2f}%) (vs flat)
ckl(mkv): {len(mkv_binary): 9} ({len(mkv_binary)/len(ckl_binary)*100:.2f}%) (vs literals)
cpso: {len(cpso_binary): 9} ({len(ckl_binary)/len(cpso_binary)*100:.2f}%)
raw: {len(raw_binary): 9} ({len(ckl_binary)/len(raw_binary)*100:.2f}%)
ckl.gz: {len(cckl_binary): 9}
ckl.gz(pin): {len(cpin_binary): 9} ({len(cpin_binary)/len(cckl_binary)*100:.2f}%) (vs flat)
ckl.gz(mkv): {len(cmkv_binary): 9} ({len(cmkv_binary)/len(cckl_binary)*100:.2f}%) (vs literals)
cpso.gz: {len(ccpso_binary): 9} ({len(cckl_binary)/len(ccpso_binary)*100:.2f}%)
raw.gz: {len(craw_binary): 9} ({len(cckl_binary)/len(craw_binary)*100:.2f}%)
""", flush=True)

N = 3
ckl{ext}: {len(cckl_binary): 9} ({len(cckl_binary)/len(craw_binary)*100:.2f}%)
ckl{ext}(pin): {len(cpin_binary): 9} ({len(cpin_binary)/len(craw_binary)*100:.2f}%)
ckl{ext}(mkv): {len(cmkv_binary): 9} ({len(cmkv_binary)/len(craw_binary)*100:.2f}%)
cpso{ext}: {len(ccpso_binary): 9} ({len(ccpso_binary)/len(craw_binary)*100:.2f}%)
raw{ext}: {len(craw_binary): 9} ({len(craw_binary)/len(craw_binary)*100:.2f}%)""", flush=True)

N = 1
shape = (256,256,64)

print("shape:", shape)
Expand All @@ -74,14 +67,6 @@ def run(labels):
labels = compresso.load("benchmarks/ws.npy.cpso.gz")
run_sample(labels, shape, N)

print("RANDOM NOISE [0,2000) uint32")
labels = np.random.randint(0,2000, size=(512,512,512), dtype=np.uint32)
run_sample(labels, shape, N)

print("BINARY NOISE [0,1] uint8 (pathological case)")
labels = np.random.randint(0,2, size=(512,512,512), dtype=np.uint8)
run_sample(labels, shape, N)

print("EMPTY")
labels = np.zeros((512,512,512), dtype=np.uint32)
run_sample(labels, shape, 1)
Expand All @@ -90,3 +75,10 @@ def run(labels):
labels = np.ones((512,512,512), dtype=np.uint32)
run_sample(labels, shape, 1)

print("RANDOM NOISE [0,2000) uint32")
labels = np.random.randint(0,2000, size=(512,512,128), dtype=np.uint32)
run_sample(labels, shape, N)

print("BINARY NOISE [0,1] uint8 (pathological case)")
labels = np.random.randint(0,2, size=(512,512,128), dtype=np.uint8)
run_sample(labels, shape, N)

0 comments on commit 64af0a1

Please sign in to comment.