feat: optimize order of entries

lib/utils/templates.js

@@ -3,9 +3,9 @@ export const URI_BASE = URI_SCHEME + '://maps.v1/'
 
 // These constants determine the file format structure
 export const STYLE_FILE = 'style.json'
-const SOURCES_FOLDER = 's'
+export const SOURCES_FOLDER = 's'
 const SPRITES_FOLDER = 'sprites'
-const FONTS_FOLDER = 'fonts'
+export const FONTS_FOLDER = 'fonts'
 
 // This must include placeholders `{z}`, `{x}`, `{y}`, since these are used to
 // define the tile URL, and this is a TileJSON standard.

lib/writer.js

@@ -14,12 +14,14 @@ import { clone } from './utils/misc.js'
 import { writeStreamFromAsync } from './utils/streams.js'
 import { replaceFontStacks } from './utils/style.js'
 import {
+  FONTS_FOLDER,
   getGlyphFilename,
   getSpriteFilename,
   getSpriteUri,
   getTileFilename,
   getTileUri,
   GLYPH_URI,
+  SOURCES_FOLDER,
   STYLE_FILE,
 } from './utils/templates.js'
 
@@ -338,11 +340,12 @@ export default class Writer extends EventEmitter {
    * This method must be called to complete the archive.
    * You must wait for your destination write stream to 'finish' before using the output.
    */
-  finish() {
+  async finish() {
     this.#prepareStyle()
     const style = JSON.stringify(this.#style)
-    this.#append(style, { name: STYLE_FILE })
-    this.#archive.finalize()
+    await this.#append(style, { name: STYLE_FILE })
+    sortEntries(this.#archive)
+    await this.#archive.finalize()
   }
 
   /**
@@ -476,3 +479,72 @@ function get2DBBox(bbox) {
   if (bbox.length === 4) return bbox
   return [bbox[0], bbox[1], bbox[3], bbox[4]]
 }
+
+/**
+ * @typedef {object} ZipEntry
+ * @property {string} name
+ */
+
+/**
+ * Dive into the internals of Archiver to sort the central directory entries of
+ * the zip, so that the style.json, ASCII glyphs, and initial tiles are listed
+ * first, which improves read speed (the map can be displayed before the entire
+ * central directory is indexed)
+ * @param {import('archiver').Archiver} archive
+ */
+function sortEntries(archive) {
+  // @ts-expect-error
+  const entries = /** @type {unknown} */ (archive._module?.engine?._entries)
+  if (!Array.isArray(entries)) {
+    throw new Error(
+      'Cannot find zip entries: check implementation changes in Archiver',
+    )
+  }
+  const sortedEntries = entries.sort(
+    /**
+     * @param {unknown} a
+     * @param {unknown} b
+     */
+    function (a, b) {
+      assertValidEntry(a)
+      assertValidEntry(b)
+      if (a.name === 'style.json') return -1
+      if (b.name === 'style.json') return 1
+      const foldersA = a.name.split('/')
+      const foldersB = b.name.split('/')
+      if (foldersA[0] === FONTS_FOLDER && foldersA[2] === '0-255.pbf.gz')
+        return -1
+      if (foldersB[0] === FONTS_FOLDER && foldersB[2] === '0-255.pbf.gz')
+        return 1
+      if (foldersA[0] === SOURCES_FOLDER && foldersB[0] !== SOURCES_FOLDER)
+        return -1
+      if (foldersB[0] === SOURCES_FOLDER && foldersA[0] !== SOURCES_FOLDER)
+        return 1
+      if (foldersA[0] === SOURCES_FOLDER && foldersB[0] === SOURCES_FOLDER) {
+        const zoomA = +foldersA[2]
+        const zoomB = +foldersB[2]
+        return zoomA - zoomB
+      }
+      return 0
+    },
+  )
+  // @ts-expect-error
+  archive._module.engine._entries = sortedEntries
+}
+
+/**
+ * @param {unknown} maybeEntry
+ * @returns {asserts maybeEntry is ZipEntry}
+ */
+function assertValidEntry(maybeEntry) {
+  if (
+    !maybeEntry ||
+    typeof maybeEntry !== 'object' ||
+    !('name' in maybeEntry) ||
+    typeof maybeEntry.name !== 'string'
+  ) {
+    throw new Error(
+      'Unexpected zip entry type: check implementation changes in Archiver',
+    )
+  }
+}

test/fixtures/valid-styles/all-types.input.json

@@ -0,0 +1,25 @@
+{
+  "version": 8,
+  "name": "Example (fake)",
+  "sources": {
+    "source1": {
+      "type": "vector",
+      "url": "https://example.com/source1.json"
+    },
+    "source2": {
+      "type": "vector",
+      "url": "https://example.com/source2.json"
+    }
+  },
+  "glyph": "https://example.com/font/{fontstack}/{range}.pbf",
+  "sprite": "https://example.com/sprite",
+  "layers": [
+    {
+      "id": "background",
+      "type": "background",
+      "paint": {
+        "background-color": "#f8f4f0"
+      }
+    }
+  ]
+}

test/write-read.js

@@ -1,7 +1,7 @@
 import SphericalMercator from '@mapbox/sphericalmercator'
 import { bbox as turfBbox } from '@turf/bbox'
 import randomStream from 'random-bytes-readable-stream'
-import { fromBuffer as zipFromBuffer } from 'yauzl-promise'
+import { fromBuffer, fromBuffer as zipFromBuffer } from 'yauzl-promise'
 
 import assert from 'node:assert/strict'
 import fs from 'node:fs/promises'
@@ -573,6 +573,85 @@ test('Raster tiles write and read', async () => {
   assert.equal(jpgTileHashOut, jpgTileHash, 'JPG tile is the same')
 })
 
+test.only('Optimized central directory order', async () => {
+  const styleInUrl = new URL(
+    './fixtures/valid-styles/all-types.input.json',
+    import.meta.url,
+  )
+
+  /** @type {import('@maplibre/maplibre-gl-style-spec').StyleSpecification} */
+  const styleIn = await readJson(styleInUrl)
+  const writer = new Writer(styleIn)
+
+  const bounds = /** @type {BBox} */ ([-40.6, -50.6, 151.6, 76.0])
+
+  for (const { x, y, z } of tileIterator({ maxzoom: 5, bounds })) {
+    for (const sourceId of ['source1', 'source2']) {
+      const stream = randomStream({ size: random(2048, 4096) }).pipe(
+        new DigestStream('md5'),
+      )
+      await writer.addTile(stream, { x, y, z, sourceId, format: 'mvt' })
+    }
+  }
+
+  for (const range of glyphRanges()) {
+    for (const font of ['font1', 'font2']) {
+      const stream = randomStream({ size: random(256, 1024) }).pipe(
+        new DigestStream('md5'),
+      )
+      await writer.addGlyphs(stream, { range, font })
+    }
+  }
+
+  const spriteImageStream = randomStream({ size: random(1024, 2048) }).pipe(
+    new DigestStream('md5'),
+  )
+  const spriteLayoutIn = {
+    airfield_11: {
+      height: 17,
+      pixelRatio: 1,
+      width: 17,
+      x: 21,
+      y: 0,
+    },
+  }
+  await writer.addSprite({
+    png: spriteImageStream,
+    json: JSON.stringify(spriteLayoutIn),
+  })
+
+  writer.finish()
+
+  const smp = await streamToBuffer(writer.outputStream)
+  const zip = await fromBuffer(smp)
+  const entries = await zip.readEntries()
+  const entriesFilenames = entries.map((e) => e.filename)
+
+  // 1. style.json
+  // 2. glyphs for 0-255 UTF codes
+  // 3. sources ordered by zoom level
+  const expectedFirstEntriesFilenames = [
+    'style.json',
+    'fonts/font2/0-255.pbf.gz',
+    'fonts/font1/0-255.pbf.gz',
+    's/0/0/0/0.mvt.gz',
+    's/1/0/0/0.mvt.gz',
+    's/0/1/0/0.mvt.gz',
+    's/1/1/0/0.mvt.gz',
+    's/0/1/0/1.mvt.gz',
+    's/1/1/0/1.mvt.gz',
+    's/0/1/1/0.mvt.gz',
+    's/1/1/1/0.mvt.gz',
+    's/0/1/1/1.mvt.gz',
+    's/1/1/1/1.mvt.gz',
+  ]
+
+  assert.deepStrictEqual(
+    entriesFilenames.slice(0, expectedFirstEntriesFilenames.length),
+    expectedFirstEntriesFilenames,
+  )
+})
+
 /**
  *
  * @param {number} min