asset: allow dynamic window size. Tweak better ratio/speed.

examples/compression/Makefile

@@ -16,19 +16,19 @@ all: compression.z64
 filesystem/%.c0: assets/%
 	@mkdir -p $(dir $@)/c0
 	@echo "    [MKASSET] $@"
-	$(N64_BINDIR)/mkasset -c 0 -o filesystem/c0 $<
+	$(N64_BINDIR)/mkasset -c 0 -w 256 -o filesystem/c0 $<
 	mv "filesystem/c0/$(notdir $<)" "$@"
 
 filesystem/%.c1: assets/%	
 	@mkdir -p $(dir $@)/c1
 	@echo "    [MKASSET] $@"
-	$(N64_BINDIR)/mkasset -c 1 -o filesystem/c1 $<
+	$(N64_BINDIR)/mkasset -c 1 -w 256 -o filesystem/c1 $<
 	mv "filesystem/c1/$(notdir $<)" "$@"
 
 filesystem/%.c2: assets/%
 	@mkdir -p $(dir $@)/c2
 	@echo "    [MKASSET] $@"
-	$(N64_BINDIR)/mkasset -c 2 -o filesystem/c2 $<
+	$(N64_BINDIR)/mkasset -c 2 -w 256 -o filesystem/c2 $<
 	mv "filesystem/c2/$(notdir $<)" "$@"
 
 $(BUILD_DIR)/compression.dfs: $(assets_conv)

src/asset.c

@@ -222,9 +222,10 @@ FILE *asset_fopen(const char *fn, int *sz)
         assertf(algos[header.algo-1].decompress_init, 
             "asset: compression level %d not initialized. Call asset_init_compression(%d) at initialization time", header.algo, header.algo);
 
-        cookie = malloc(sizeof(cookie_cmp_t) + algos[header.algo-1].state_size);
+        int winsize = asset_winsize_from_flags(header.flags);
+        cookie = malloc(sizeof(cookie_cmp_t) + algos[header.algo-1].state_size + winsize);
         cookie->read = algos[header.algo-1].decompress_read;
-        algos[header.algo-1].decompress_init(cookie->state, f);
+        algos[header.algo-1].decompress_init(cookie->state, f, winsize);
 
         cookie->fp = f;
         cookie->pos = 0;

src/asset_internal.h

@@ -4,14 +4,45 @@
 #include <stdint.h>
 #include <stdio.h>
 
-#define ASSET_MAGIC    "DCA"   ///< Magic compressed asset header
+#define ASSET_MAGIC                 "DCA"   ///< Magic compressed asset header
+#define ASSET_FLAG_WINSIZE_MASK     0x0007  ///< Mask to isolate the window size in the flags
+#define ASSET_FLAG_WINSIZE_16K      0x0000  ///< 16 KiB window size
+#define ASSET_FLAG_WINSIZE_8K       0x0001  ///< 8 KiB window size
+#define ASSET_FLAG_WINSIZE_4K       0x0002  ///< 4 KiB window size
+#define ASSET_FLAG_WINSIZE_2K       0x0003  ///< 2 KiB window size
+#define ASSET_FLAG_WINSIZE_32K      0x0004  ///< 32 KiB window size
+#define ASSET_FLAG_WINSIZE_64K      0x0005  ///< 64 KiB window size
+#define ASSET_FLAG_WINSIZE_128K     0x0006  ///< 128 KiB window size
+#define ASSET_FLAG_WINSIZE_256K     0x0007  ///< 256 KiB window size
+
+__attribute__((used))
+static inline int asset_winsize_from_flags(uint16_t flags) {
+    flags &= ASSET_FLAG_WINSIZE_MASK;
+    if (flags & 4) 
+        return (2*1024) << flags;
+    else
+        return (16*1024) >> flags;
+}
+
+__attribute__((used))
+static int asset_winsize_to_flags(int winsize) {
+    if (winsize == 16*1024)  return ASSET_FLAG_WINSIZE_16K;
+    if (winsize == 8*1024)   return ASSET_FLAG_WINSIZE_8K;
+    if (winsize == 4*1024)   return ASSET_FLAG_WINSIZE_4K;
+    if (winsize == 2*1024)   return ASSET_FLAG_WINSIZE_2K;
+    if (winsize == 32*1024)  return ASSET_FLAG_WINSIZE_32K;
+    if (winsize == 64*1024)  return ASSET_FLAG_WINSIZE_64K;
+    if (winsize == 128*1024) return ASSET_FLAG_WINSIZE_128K;
+    if (winsize == 256*1024) return ASSET_FLAG_WINSIZE_256K;
+    return -1;
+}
 
 /** @brief Header of a compressed asset */
 typedef struct {
     char magic[3];          ///< Magic header
     uint8_t version;        ///< Version of the asset header
     uint16_t algo;          ///< Compression algorithm
-    uint16_t flags;         ///< Flags (unused for now)
+    uint16_t flags;         ///< Flags
     uint32_t cmp_size;      ///< Compressed size in bytes
     uint32_t orig_size;     ///< Original size in bytes
 } asset_header_t;
@@ -20,10 +51,11 @@ _Static_assert(sizeof(asset_header_t) == 16, "invalid sizeof(asset_header_t)");
 
 /** @brief A decompression algorithm used by the asset library */
 typedef struct {
-    int state_size;     ///< Size of the decompression state
+    int state_size;     ///< Basic size of the decompression state (without ringbuffer)
 
     /** @brief Initialize the decompression state */
-    void (*decompress_init)(void *state, FILE *fp);
+    void (*decompress_init)(void *state, FILE *fp, int winsize);
+
     /** @brief Partially read a decompressed file from a state */
     ssize_t (*decompress_read)(void *state, void *buf, size_t len);
 

src/audio/ym64.c

@@ -124,9 +124,9 @@ void ym64player_open(ym64player_t *player, const char *fn, ym64player_songinfo_t
 
 		// Initialize decompressor and re-read the header (this time, it will
 		// be decompressed and we should find a valid YM header).
-		player->decoder = malloc(DECOMPRESS_LZH5_STATE_SIZE);
+		player->decoder = malloc(DECOMPRESS_LZH5_STATE_SIZE + DECOMPRESS_LZH5_DEFAULT_WINDOW_SIZE);
 		offset = 0;
-		decompress_lzh5_init(player->decoder, player->f);
+		decompress_lzh5_init(player->decoder, player->f, DECOMPRESS_LZH5_DEFAULT_WINDOW_SIZE);
 		_ymread(head, 12);
 	}
 

src/compress/lz4_dec.c

@@ -15,9 +15,6 @@
 #endif
 
 #define MIN_MATCH_SIZE  4
-#define MIN_OFFSET 1
-#define MAX_OFFSET 0xffff
-#define HISTORY_SIZE 65536
 #define LITERALS_RUN_LEN 15
 #define MATCH_RUN_LEN 15
 
@@ -283,15 +280,15 @@ static void lz4_read(lz4dec_state_t *lz4, void *buf, size_t len)
    }
 }
 
-void decompress_lz4_init(void *state, FILE *fp)
+void decompress_lz4_init(void *state, FILE *fp, int winsize)
 {
    lz4dec_state_t *lz4 = (lz4dec_state_t*)state;
    lz4->fp = fp;
    lz4->eof = false;
    lz4->buf_idx = 0;
    lz4->buf_size = 0;
    memset(&lz4->st, 0, sizeof(lz4->st));
-   __ringbuf_init(&lz4->ringbuf);
+   __ringbuf_init(&lz4->ringbuf, state+sizeof(lz4dec_state_t), winsize);
 }
 
 ssize_t decompress_lz4_read(void *state, void *buf, size_t len)

src/compress/lz4_dec_internal.h

@@ -59,9 +59,9 @@ int decompress_lz4_full_mem(const unsigned char *src, int src_size,
     unsigned char *dst, int dst_size, bool dma_race);
 
 
-#define DECOMPRESS_LZ4_STATE_SIZE  (16552)
+#define DECOMPRESS_LZ4_STATE_SIZE  176
 
-void decompress_lz4_init(void *state, FILE *fp);
+void decompress_lz4_init(void *state, FILE *fp, int winsize);
 ssize_t decompress_lz4_read(void *state, void *buf, size_t len);
 void* decompress_lz4_full(const char *fn, FILE *fp, size_t cmp_size, size_t size);
 

src/compress/lzh5.c

@@ -423,12 +423,12 @@ static int lha_lh_new_init(LHANewDecoder *decoder, FILE *fp, uint32_t rom_addr)
 	return 1;
 }
 
-static int lha_lh_new_init_partial(LHANewDecoderPartial *decoder, FILE *fp)
+static int lha_lh_new_init_partial(LHANewDecoderPartial *decoder, FILE *fp, void *window, int winsize)
 {
 	lha_lh_new_init(&decoder->decoder, fp, 0);
 
 	// Initialize data structures.
-	__ringbuf_init(&decoder->ringbuf);
+	__ringbuf_init(&decoder->ringbuf, window, winsize);
 	decoder->ringbuf_copy_offset = 0;
 	decoder->ringbuf_copy_count = 0;
 
@@ -532,10 +532,10 @@ static size_t lha_lh_new_read_full(LHANewDecoder *decoder, uint8_t *buf, int sz)
 
 _Static_assert(sizeof(LHANewDecoderPartial) <= DECOMPRESS_LZH5_STATE_SIZE, "LZH5 state size is wrong");
 
-void decompress_lzh5_init(void *state, FILE *fp)
+void decompress_lzh5_init(void *state, FILE *fp, int winsize)
 {
 	LHANewDecoderPartial *decoder = (LHANewDecoderPartial *)state;
-	lha_lh_new_init_partial(decoder, fp);
+	lha_lh_new_init_partial(decoder, fp, state+sizeof(LHANewDecoderPartial), winsize);
 }
 
 ssize_t decompress_lzh5_read(void *state, void *buf, size_t len)

src/compress/lzh5_internal.h

@@ -14,9 +14,10 @@ extern "C" {
  * Note that this can still be allocated on the stack, as the stack size
  * configured by libdragon is 64KB.
  */
-#define DECOMPRESS_LZH5_STATE_SIZE    (22464+16)
+#define DECOMPRESS_LZH5_STATE_SIZE            (6096+16)
+#define DECOMPRESS_LZH5_DEFAULT_WINDOW_SIZE   (8192)
 
-void decompress_lzh5_init(void *state, FILE *fp);
+void decompress_lzh5_init(void *state, FILE *fp, int winsize);
 ssize_t decompress_lzh5_read(void *state, void *buf, size_t len);
 int decompress_lzh5_pos(void *state);
 

src/compress/ringbuf.c

@@ -1,31 +1,36 @@
 #include "ringbuf_internal.h"
 #include "../utils.h"
+#include <assert.h>
 
-void __ringbuf_init(decompress_ringbuf_t *ringbuf)
+void __ringbuf_init(decompress_ringbuf_t *ringbuf, uint8_t *buf, int size)
 {
+    assert(size > 0);
+    assert((size & (size-1)) == 0); // check if power of two
+    ringbuf->ringbuf = buf;
+    ringbuf->ringbuf_size = size ;
     ringbuf->ringbuf_pos = 0;
 }
 
 void __ringbuf_write(decompress_ringbuf_t *ringbuf, uint8_t *src, int count)
 {
     while (count > 0) {
-        int n = MIN(count, RING_BUFFER_SIZE - ringbuf->ringbuf_pos);
+        int n = MIN(count, ringbuf->ringbuf_size - ringbuf->ringbuf_pos);
         memcpy(ringbuf->ringbuf + ringbuf->ringbuf_pos, src, n);
         ringbuf->ringbuf_pos += n;
-        ringbuf->ringbuf_pos &= RING_BUFFER_SIZE-1;
+        ringbuf->ringbuf_pos &= ringbuf->ringbuf_size - 1;
         src += n;
         count -= n;
     }
 }
 
 void __ringbuf_copy(decompress_ringbuf_t *ringbuf, int copy_offset, uint8_t *dst, int count)
 {
-    int ringbuf_copy_pos = (ringbuf->ringbuf_pos - copy_offset) & (RING_BUFFER_SIZE-1);
+    int ringbuf_copy_pos = (ringbuf->ringbuf_pos - copy_offset) & (ringbuf->ringbuf_size - 1);
     int dst_pos = 0;
     while (count > 0) {
         int wn = count;
-        wn = wn < RING_BUFFER_SIZE - ringbuf_copy_pos     ? wn : RING_BUFFER_SIZE - ringbuf_copy_pos;
-        wn = wn < RING_BUFFER_SIZE - ringbuf->ringbuf_pos ? wn : RING_BUFFER_SIZE - ringbuf->ringbuf_pos;
+        wn = wn < ringbuf->ringbuf_size - ringbuf_copy_pos     ? wn : ringbuf->ringbuf_size - ringbuf_copy_pos;
+        wn = wn < ringbuf->ringbuf_size - ringbuf->ringbuf_pos ? wn : ringbuf->ringbuf_size - ringbuf->ringbuf_pos;
         count -= wn;
 
         // Check if there's an overlap in the ring buffer between read and write pos, in which
@@ -58,7 +63,7 @@ void __ringbuf_copy(decompress_ringbuf_t *ringbuf, int copy_offset, uint8_t *dst
             wn -= 1;
         }
 
-        ringbuf_copy_pos %= RING_BUFFER_SIZE;
-        ringbuf->ringbuf_pos %= RING_BUFFER_SIZE;
+        ringbuf_copy_pos &= ringbuf->ringbuf_size - 1;
+        ringbuf->ringbuf_pos &= ringbuf->ringbuf_size - 1;
     }
 }

src/compress/ringbuf_internal.h

@@ -3,26 +3,22 @@
 
 #include <stdint.h>
 
-///< Size of the ring buffer in bytes. This happens to work for both lz4 and lzh5
-#ifndef RING_BUFFER_SIZE
-#define RING_BUFFER_SIZE    (16 * 1024)
-#endif
-
 /**
  * @brief A ring buffer used for streaming decompression.
  */
 typedef struct {
-	uint8_t ringbuf[RING_BUFFER_SIZE];      ///< The ring buffer itself
+	uint8_t* ringbuf;                       ///< The ring buffer itself
+    unsigned int ringbuf_size;              ///< Size of the ring buffer (power of two)
 	unsigned int ringbuf_pos;               ///< Current write position in the ring buffer
 } decompress_ringbuf_t;
 
 
-void __ringbuf_init(decompress_ringbuf_t *ringbuf);
+void __ringbuf_init(decompress_ringbuf_t *ringbuf, uint8_t *buf, int winsize);
 
 inline void __ringbuf_writebyte(decompress_ringbuf_t *ringbuf, uint8_t byte)
 {
     ringbuf->ringbuf[ringbuf->ringbuf_pos++] = byte;
-    ringbuf->ringbuf_pos &= (RING_BUFFER_SIZE - 1);
+    ringbuf->ringbuf_pos &= ringbuf->ringbuf_size-1;
 }
 
 /**

tools/audioconv64/conv_ym64.c

@@ -158,7 +158,7 @@ int ym_convert(const char *infn, const char *outfn) {
         // https://github.com/fragglet/lhasa, stored in lzh5.h.
         fseek(ym_f, head[0]+2, SEEK_SET);
         ym_compressed = true;
-        decompress_lzh5_init(ym_decoder, ym_f);
+        decompress_lzh5_init(ym_decoder, ym_f, DECOMPRESS_LZH5_DEFAULT_WINDOW_SIZE);
         ymread(head, 12);
     }