improve io

src/iers.test.ts

@@ -4,7 +4,7 @@ import { iersa, iersb } from './iers'
 import { Timescale, timeYMDHMS } from './time'
 
 test('iersA', async () => {
-	await iersa.load(await Bun.file('data/finals2000A.txt').arrayBuffer())
+	await iersa.load(Bun.file('data/finals2000A.txt').stream())
 	let t = timeYMDHMS(2020, 10, 7, 12, 34, 56, Timescale.UTC)
 	expect(iersa.delta(t)).toBe(-0.17181135242592593)
 	expect(iersa.xy(t)).toEqual([arcsec(0.1878143362962963), arcsec(0.3180433324074074)])
@@ -13,13 +13,13 @@ test('iersA', async () => {
 	expect(iersa.delta(t)).toBe(0.0862207)
 	expect(iersa.xy(t)).toEqual([arcsec(0.094347), arcsec(0.293316)])
 
-    t = timeYMDHMS(1900, 10, 7, 12, 34, 56, Timescale.UTC)
+	t = timeYMDHMS(1900, 10, 7, 12, 34, 56, Timescale.UTC)
 	expect(iersa.delta(t)).toBe(0.8075)
 	expect(iersa.xy(t)).toEqual([arcsec(0.143), arcsec(0.137)])
 })
 
 test('iersB', async () => {
-	await iersb.load(await Bun.file('data/eopc04.1962-now.txt').arrayBuffer())
+	await iersb.load(Bun.file('data/eopc04.1962-now.txt').stream())
 	let t = timeYMDHMS(2020, 10, 7, 12, 34, 56, Timescale.UTC)
 	expect(iersb.delta(t)).toBe(-0.17180533112962962)
 	expect(iersb.xy(t)).toEqual([arcsec(0.1878133848148148), arcsec(0.3179746625925926)])
@@ -28,7 +28,7 @@ test('iersB', async () => {
 	expect(iersb.delta(t)).toBe(0.0523072)
 	expect(iersb.xy(t)).toEqual([arcsec(0.202982), arcsec(0.338377)])
 
-    t = timeYMDHMS(1900, 10, 7, 12, 34, 56, Timescale.UTC)
+	t = timeYMDHMS(1900, 10, 7, 12, 34, 56, Timescale.UTC)
 	expect(iersb.delta(t)).toBe(0.0326338)
 	expect(iersb.xy(t)).toEqual([arcsec(-0.0127), arcsec(0.213)])
 })

src/iers.ts

@@ -1,14 +1,14 @@
 import { arcsec, type Angle } from './angle'
 import { MJD0 } from './constants'
 import { binarySearch } from './helper'
-import { readLinesFromArrayBuffer } from './io'
+import { readableStreamToLines } from './io'
 import type { PolarMotion, Time, TimeDelta } from './time'
 
 export interface Iers {
 	delta: TimeDelta
 	xy: PolarMotion
 
-	load: (buffer: AllowSharedBufferSource) => Promise<void>
+	load: (stream: ReadableStream<Uint8Array>) => Promise<void>
 	clear: () => void
 }
 
@@ -68,7 +68,7 @@ export abstract class IersBase implements Iers {
 		}
 	}
 
-	abstract load(buffer: AllowSharedBufferSource): Promise<void>
+	abstract load(stream: ReadableStream<Uint8Array>): Promise<void>
 
 	clear() {
 		this.mjd = []
@@ -81,10 +81,10 @@ export abstract class IersBase implements Iers {
 // https://datacenter.iers.org/data/9/finals2000A.all
 // https://maia.usno.navy.mil/ser7/readme.finals2000A
 export class IersA extends IersBase {
-	load(buffer: AllowSharedBufferSource) {
+	async load(stream: ReadableStream<Uint8Array>) {
 		this.clear()
 
-		return readLinesFromArrayBuffer(buffer, (line) => {
+		for await (const line of readableStreamToLines(stream)) {
 			const pmXa = parseFloat(line.substring(18, 27).trim())
 			const pmYa = parseFloat(line.substring(37, 46).trim())
 			const pmXb = parseFloat(line.substring(134, 144).trim())
@@ -99,18 +99,18 @@ export class IersA extends IersBase {
 				this.pmY.push(pmYb || pmYa)
 				this.dut1.push(dut1b || dut1a)
 			}
-		})
+		}
 	}
 }
 
 // https://hpiers.obspm.fr/iers/eop/eopc04/eopc04.1962-now
 // https://hpiers.obspm.fr/eoppc/eop/eopc04/eopc04.txt
 export class IersB extends IersBase {
-	async load(buffer: AllowSharedBufferSource) {
+	async load(stream: ReadableStream<Uint8Array>) {
 		this.clear()
 
-		return readLinesFromArrayBuffer(buffer, (line) => {
-			if (line.startsWith('#')) return
+		for await (const line of readableStreamToLines(stream)) {
+			if (line.startsWith('#')) continue
 
 			const pmX = parseFloat(line.substring(26, 38).trim())
 			const pmY = parseFloat(line.substring(38, 50).trim())
@@ -123,7 +123,7 @@ export class IersB extends IersBase {
 				this.pmY.push(pmY)
 				this.dut1.push(dut1)
 			}
-		})
+		}
 	}
 }
 
@@ -143,8 +143,7 @@ export class IersAB implements Iers {
 		return b
 	}
 
-	// eslint-disable-next-line @typescript-eslint/no-unused-vars
-	load(buffer: AllowSharedBufferSource): Promise<void> {
+	load(stream: ReadableStream<Uint8Array>): Promise<void> {
 		throw new Error('not supported')
 	}
 

src/io.test.ts

@@ -1,13 +1,25 @@
 import { expect, test } from 'bun:test'
-import { readLinesFromArrayBuffer } from './io'
+import { arrayBufferToLines, readableStreamToLines } from './io'
 
-test('readLinesFromArrayBuffer', async () => {
+test('arrayBufferToLines', async () => {
 	const blob = new Blob(['line 1\n', 'line 2\n', '', 'line 3\nline 4\n\n'])
 	const lines: string[] = []
 
-	await readLinesFromArrayBuffer(await blob.arrayBuffer(), (line) => {
+	for await (const line of arrayBufferToLines(await blob.arrayBuffer())) {
 		lines.push(line)
-	})
+	}
+
+	expect(lines).toHaveLength(4)
+	expect(lines).toContainAllValues(['line 1', 'line 2', 'line 3', 'line 4'])
+})
+
+test('readableStreamToLines', async () => {
+	const blob = new Blob(['line 1\n', 'line 2\n', '', 'line 3\nline 4\n\n'])
+	const lines: string[] = []
+
+	for await (const line of readableStreamToLines(blob.stream())) {
+		lines.push(line)
+	}
 
 	expect(lines).toHaveLength(4)
 	expect(lines).toContainAllValues(['line 1', 'line 2', 'line 3', 'line 4'])

src/io.ts

@@ -1,4 +1,4 @@
-export async function readLinesFromArrayBuffer(buffer: AllowSharedBufferSource, callback: (line: string) => void, charset: string = 'utf-8') {
+export async function* arrayBufferToLines(buffer: AllowSharedBufferSource, charset: string = 'utf-8') {
 	const decoder = new TextDecoder(charset)
 	const stream = new ReadableStream<AllowSharedBufferSource>({
 		start(controller) {
@@ -22,7 +22,7 @@ export async function readLinesFromArrayBuffer(buffer: AllowSharedBufferSource,
 		// Store all complete lines
 		for (let i = 0; i < parts.length - 1; i++) {
 			if (parts[i]) {
-				callback(parts[i])
+				yield parts[i]
 			}
 		}
 
@@ -34,6 +34,34 @@ export async function readLinesFromArrayBuffer(buffer: AllowSharedBufferSource,
 	}
 
 	if (line) {
-		callback(line)
+		yield line
+	}
+}
+
+export async function* readableStreamToLines(stream: ReadableStream<Uint8Array>, charset: string = 'utf-8') {
+	const decoder = new TextDecoder(charset)
+
+	let line = ''
+
+	for await (const chunk of stream) {
+		// Decode incrementally
+		line = decoder.decode(chunk, { stream: true })
+
+		// Split by newline
+		const parts = line.split('\n')
+
+		// Store all complete lines
+		for (let i = 0; i < parts.length - 1; i++) {
+			if (parts[i]) {
+				yield parts[i]
+			}
+		}
+
+		// Keep the remainder (last part) for the next chunk
+		line = parts[parts.length - 1]
+	}
+
+	if (line) {
+		yield line
 	}
 }

src/location.test.ts

@@ -6,7 +6,7 @@ import { Ellipsoid, geocentric, geodetic, lst, polarRadius } from './location'
 import { Timescale, timeYMDHMS } from './time'
 
 beforeAll(async () => {
-	await iersb.load(await Bun.file('data/eopc04.1962-now.txt').arrayBuffer())
+	await iersb.load(Bun.file('data/eopc04.1962-now.txt').stream())
 })
 
 test('lst', () => {

src/time.test.ts

@@ -2,7 +2,7 @@ import { beforeAll, expect, test, type CustomMatcher } from 'bun:test'
 import { deg, hour } from './angle'
 import { J2000 } from './constants'
 import { meter } from './distance'
-import { iersa, iersb } from './iers'
+import { iersb } from './iers'
 import { Ellipsoid, geodetic } from './location'
 import { equationOfOrigins, era, gast, gmst, meanObliquity, normalize, nutation, precession, precessionNutation, tai, tcb, tcg, tdb, tdbMinusTt, tdbMinusTtByFairheadAndBretagnon1990, time, timeBesselian, timeGPS, timeJulian, timeMJD, Timescale, timeUnix, timeYMDHMS, tt, ut1, utc, type Time } from './time'
 
@@ -33,8 +33,7 @@ declare module 'bun:test' {
 }
 
 beforeAll(async () => {
-	await iersa.load(await Bun.file('data/finals2000A.txt').arrayBuffer())
-	await iersb.load(await Bun.file('data/eopc04.1962-now.txt').arrayBuffer())
+	await iersb.load(Bun.file('data/eopc04.1962-now.txt').stream())
 })
 
 test('time', () => {

tsconfig.json

@@ -1,6 +1,6 @@
 {
 	"compilerOptions": {
-		"lib": ["ESNext", "DOM"],
+		"lib": ["ESNext", "DOM", "DOM.AsyncIterable"],
 		"target": "ESNext",
 		"module": "ESNext",
 		"moduleDetection": "force",