-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
121 lines (101 loc) · 6.04 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
SHELL = /bin/bash
SIZE != <webster wc -c
COMPRESSORS = lz4 gzip zstd bzip2 brotli xz lzip
STATS = $(patsubst %,tables/%.tsv,$(COMPRESSORS))
all: $(STATS) clean
# Download the Webster collection from the Silesia corpus: http://sun.aei.polsl.pl/~sdeor/index.php?page=silesia
webster:
curl -s http://sun.aei.polsl.pl/~sdeor/corpus/webster.bz2 -O
<webster.bz2 bzip2 -d >webster
rm webster.bz2
# ratio: size of uncompressed data for every MB of compressed payload.
# compression: average number of MB that can get compressed every second.
# decompression: average number of MB that can get decompressed every second.
# load time: assuming a 1 MB/s Internet connection, how much time does it
# take to transmit 1 MB of raw data by transmitting the (already-compressed)
# payload, and uncompressing it on the client?
# send time: same as load time, including compression time.
tables/stats.tsv: webster $(STATS)
stats.tsv-header:
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >tables/stats.tsv
tables/lz4.tsv:
compressor=lz4; file=/tmp/webster.lz4; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq -12 -1`; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
tables/gzip.tsv:
compressor=gzip; file=/tmp/webster.gz; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq -9 -1`; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
tables/zstd.tsv:
compressor=zstd; file=/tmp/webster.zst; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq -22 -1`; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor --ultra $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
tables/bzip2.tsv:
compressor=bzip2; file=/tmp/webster.bz2; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq -9 -1`; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
tables/brotli.tsv:
compressor=brotli; file=/tmp/webster.br; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq 0 11`; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor -q $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
tables/xz.tsv:
compressor=xz; file=/tmp/webster.xz; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq -9 -1` -0; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
tables/lzip.tsv:
compressor=lzip; file=/tmp/webster.lz; \
echo -e 'Compressor\tRatio\tCompression (MB/s)\tDecompression (MB/s)\tLoad time (s/MB)\tSend time (s/MB)' >$@; \
for l in `seq -9 -1` -0; do \
comp="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <webster $$compressor $$l 2>&1 >$$file)")"; \
ratio="$$(bc <<<"scale=3; $(SIZE)/$$(<$$file wc -c)")"; \
dec="$$(bc <<<"scale=3; $(SIZE)/1000000/$$(/usr/bin/time -f '%U' <$$file $$compressor -d 2>&1 >/dev/null)")"; \
echo -e "$$compressor $$l\t$$ratio\t$$comp\t$$dec\t$$(bc <<<"scale=3;1/$$ratio+1/$$dec")\t$$(bc <<<"scale=3;1/$$comp+1/$$ratio+1/$$dec")" >>$@; \
done
plots/sending.svg: $(STATS) plots sending.plot
gnuplot sending.plot >$@
plots/loading.svg: $(STATS) plots loading.plot
gnuplot loading.plot >$@
plots/compression.svg: $(STATS) plots compression.plot
for f in tables/*; do \
<$$f tail -n +2 | sort -k2 -t$$'\t' -r | head -1; \
done | sort -k2 -t$$'\t' | \
gnuplot compression.plot >$@
plots/compression-speed.svg: $(STATS) plots compression-speed.plot
gnuplot compression-speed.plot >$@
tables:
mkdir -p tables
plots:
mkdir -p plots
clean:
rm -f /tmp/webster.*
.PHONY: all stats.tsv-header clean