Merge pull request #167 from StarfilesFileSharing/alpha

Alpha

deno.jsonc

@@ -1,6 +1,6 @@
 {
 	"name": "@starfiles/hydrafiles",
-	"version": "0.8",
+	"version": "0.8.1",
 	"description": "The (P2P) web privacy layer.",
 	"main": "src/hydrafiles.ts",
 	"exports": {

src/config.ts

@@ -27,7 +27,7 @@ export interface Config {
 	/**
 	 * How often to log client state (in milliseconds).
 	 * -1 to disable.
-	 * @default -1
+	 * @default 30000
 	 */
 	summarySpeed: number;
 
@@ -173,7 +173,7 @@ const defaultConfig: Config = {
 	"timeout": 60000,
 	"uploadSecret": "",
 	"logLevel": "normal",
-	"summarySpeed": -1,
+	"summarySpeed": 300000,
 	"backfill": true,
 	"comparePeersSpeed": 3600000,
 	"compareFilesSpeed": 300000,

src/file.ts

@@ -4,6 +4,7 @@ import type { indexedDB } from "https://deno.land/x/[email protected]/ponyfill.ts
 import { join } from "https://deno.land/[email protected]/path/mod.ts";
 import type { Database } from "jsr:@db/sqlite";
 import type { EthAddress } from "./wallet.ts";
+import { delay } from "https://deno.land/[email protected]/async/delay.ts";
 
 type DatabaseWrapper = { type: "UNDEFINED"; db: undefined } | { type: "SQLITE"; db: Database } | { type: "INDEXEDDB"; db: IDBDatabase };
 
@@ -721,7 +722,7 @@ export class File implements FileAttributes {
 			console.log(`  ${this.hash}  Valid hash`);
 
 			const ethAddress = response.headers.get("Ethereum-Address");
-			if (ethAddress) this._client.wallet.transfer(ethAddress as EthAddress, 0.0001);
+			if (ethAddress) this._client.wallet.transfer(ethAddress as EthAddress, 0.00001);
 
 			if (this.name === null || this.name.length === 0) {
 				this.name = String(response.headers.get("Content-Disposition")?.split("=")[1].replace(/"/g, "").replace(" [HYDRAFILES]", ""));
@@ -763,22 +764,24 @@ class Files {
 		return file;
 	}
 
-	backfillFiles = async (): Promise<void> => {
-		while (true) {
-			try {
+	backfillFiles = (): void => {
+		setTimeout(async () => {
+			while (true) {
+				console.log("Backfilling file");
 				const keys = Array.from(this.files.keys());
-				if (keys.length === 0) return;
+				if (keys.length === 0) {
+					await delay(500);
+					continue;
+				}
 				const randomKey = keys[Math.floor(Math.random() * keys.length)];
 				const file = this.files.get(randomKey);
-				if (!file) return;
+				if (!file) continue;
 				if (file) {
 					console.log(`  ${file.hash}  Backfilling file`);
 					await file.getFile({ logDownloads: false });
 				}
-			} catch (e) {
-				if (this._client.config.logLevel === "verbose") throw e;
 			}
-		}
+		}, 2000); // Run 2 secs late because of Files construct being async
 	};
 
 	// TODO: Compare list between all peers and give score based on how similar they are. 100% = all exactly the same, 0% = no items in list were shared. The lower the score, the lower the propagation times, the lower the decentralisation

src/rpc/peers/http.ts

@@ -385,7 +385,7 @@ class HTTPPeer implements PeerAttributes {
 			console.log(`  ${hash}  Valid hash`);
 
 			const ethAddress = response.headers.get("Ethereum-Address");
-			if (ethAddress) this._client.wallet.transfer(ethAddress as EthAddress, 0.0001);
+			if (ethAddress) this._client.wallet.transfer(ethAddress as EthAddress, 0.00001);
 
 			if (file.name === undefined || file.name === null || file.name.length === 0) {
 				file.name = String(response.headers.get("Content-Disposition")?.split("=")[1].replace(/"/g, "").replace(" [HYDRAFILES]", ""));
@@ -516,6 +516,7 @@ export default class HTTPPeers {
 				if (response instanceof Response) {
 					const remotePeers = (await response.json()) as HTTPPeer[];
 					for (const remotePeer of remotePeers) {
+						if (Utils.isPrivateIP(remotePeer.host)) continue;
 						this.add(remotePeer.host).catch((e) => {
 							if (this._rpcClient._client.config.logLevel === "verbose") console.error(e);
 						});

src/rpc/peers/rtc.ts

@@ -26,6 +26,8 @@ export type SignallingMessage = SignallingAnnounce | SignallingOffer | Signallin
 type PeerConnection = { conn: RTCPeerConnection; channel: RTCDataChannel; startTime: number };
 type PeerConnections = { [id: string]: { offered?: PeerConnection; answered?: PeerConnection } };
 
+const receivedPackets: Record<string, string[]> = {};
+
 function arrayBufferToUnicodeString(buffer: ArrayBuffer): string {
 	const uint16Array = new Uint16Array(buffer);
 	const chunkSize = 10000;
@@ -107,16 +109,32 @@ class RTCPeers {
 			const { id, url, ...data } = JSON.parse(e.data as string);
 			const req = new Request(url, data);
 			const response = await this._rpcClient._client.rpcServer.handleRequest(req);
-			const headersObj: Record<string, string> = {};
+			const headers: Record<string, string> = {};
 			response.headers.forEach((value, key) => {
-				headersObj[key] = value;
+				headers[key] = value;
 			});
 			const body = arrayBufferToUnicodeString(new Uint8Array(await response.arrayBuffer()));
 			const status = response.status;
 			const statusText = response.statusText;
 
 			console.log(`WebRTC: (11/12): Sending response`);
-			channel.send(JSON.stringify({ body, status, statusText, headers: headersObj, id }));
+			const message = JSON.stringify({ body, status, statusText, headers, id });
+			channel.send(message);
+
+			const maxPacketSize = 8 * 1024;
+			const total = Math.ceil(message.length / maxPacketSize);
+
+			for (let i = 0; i < total; i++) {
+				const start = i * maxPacketSize;
+				const end = start + maxPacketSize;
+				const packet = {
+					id,
+					i,
+					total,
+					body: message.slice(start, end),
+				};
+				channel.send(JSON.stringify(packet));
+			}
 		};
 		conn.addEventListener("iceconnectionstatechange", () => {
 			if (conn.iceConnectionState === "disconnected" || conn.iceConnectionState === "closed" || conn.iceConnectionState === "failed") {
@@ -286,6 +304,18 @@ class RTCPeers {
 
 			const responsePromise = new Promise<Response>((resolve, reject) => {
 				connection.channel.onmessage = (e) => {
+					const packet = JSON.parse(e.data as string);
+
+					if (!receivedPackets[packet.id]) receivedPackets[packet.id] = [];
+					receivedPackets[packet.id][packet.index] = packet.body;
+
+					if (receivedPackets[packet.id].filter(Boolean).length === packet.total) {
+						const message = receivedPackets[packet.id].join("");
+						delete receivedPackets[packet.id];
+						const fullMessage = JSON.parse(message);
+						console.log("Received full message:", fullMessage);
+					}
+
 					try {
 						const { id, status, statusText, headers, body } = JSON.parse(e.data as string);
 						console.log(`WebRTC: (12/12):  ${id}  Received response`);

src/utils.ts

@@ -21,7 +21,7 @@ class Utils {
 	static hashUint8Array = async (uint8Array: Uint8Array): Promise<Sha256> => encodeHex(await crypto.subtle.digest("SHA-256", uint8Array)) as Sha256;
 	static isValidInfoHash = (hash: string): boolean => /^[a-f0-9]{40}$/.test(hash);
 	static isIp = (host: string): boolean => /^https?:\/\/(?:\d+\.){3}\d+(?::\d+)?$/.test(host);
-	static isPrivateIP = (ip: string): boolean => /^https?:\/\/(?:10\.|(?:172\.(?:1[6-9]|2\d|3[0-1]))\.|192\.168\.|169\.254\.|127\.|224\.0\.0\.|255\.255\.255\.255)/.test(ip);
+	static isPrivateIP = (ip: string): boolean => /^https?:\/\/(?:10\.|(?:172\.(?:1[6-9]|2\d|3[0-1]))\.|192\.168\.|169\.254\.|127\.|224\.0\.0\.|255\.255\.255\.255|localhost)/.test(ip);
 	static interfere = (signalStrength: number): number => signalStrength >= 95 ? this.getRandomNumber(90, 100) : Math.ceil(signalStrength * (1 - (this.getRandomNumber(0, 10) / 100)));
 	remainingStorage = async (): Promise<number> => this._config.maxCache - await this.calculateUsedStorage();
 	static createNonNegativeNumber = (n: number): NonNegativeNumber => (Number.isInteger(n) && n >= 0 ? n : 0) as NonNegativeNumber;

src/wallet.ts

@@ -46,6 +46,7 @@ class Wallet {
 	}
 
 	public async transfer(to: EthAddress, amount: number): Promise<void> {
+		console.log(`Transferring ${amount} to ${to}`);
 		const hash = await this.client.sendTransaction({
 			account: this.account,
 			chain: sepolia,