Merge pull request #141 from StarfilesFileSharing/alpha

Alpha

build.ts

@@ -1,6 +1,18 @@
 import * as esbuild from "npm:esbuild";
 import { denoPlugins } from "jsr:@luca/esbuild-deno-loader";
 
-console.log(await esbuild.build({ plugins: [...denoPlugins()], entryPoints: ["./src/hydrafiles.ts"], outfile: "./build/hydrafiles-web.esm.js", bundle: true, format: "esm", platform: "browser", sourcemap: true }));
+console.log(
+	await esbuild.build({
+		plugins: [...denoPlugins()],
+		entryPoints: ["./src/hydrafiles.ts"],
+		outfile: "./build/hydrafiles-web.esm.js",
+		bundle: true,
+		format: "esm",
+		platform: "browser",
+		sourcemap: true,
+		minify: true,
+		treeShaking: true,
+	}),
+);
 
 esbuild.stop();

deno.jsonc

@@ -1,6 +1,6 @@
 {
 	"name": "@starfiles/hydrafiles",
-	"version": "0.7.30",
+	"version": "0.7.31",
 	"description": "The web privacy layer.",
 	"main": "src/hydrafiles.ts",
 	"exports": {

src/file.ts

@@ -2,7 +2,7 @@ import type Hydrafiles from "./hydrafiles.ts";
 import Utils, { type NonNegativeNumber } from "./utils.ts";
 import type { indexedDB } from "https://deno.land/x/[email protected]/ponyfill.ts";
 import { join } from "https://deno.land/[email protected]/path/mod.ts";
-import type { Database } from "jsr:@db/sqlite@0.11";
+import type { Database } from "jsr:@db/sqlite";
 
 type DatabaseWrapper = { type: "UNDEFINED"; db: undefined } | { type: "SQLITE"; db: Database } | { type: "INDEXEDDB"; db: IDBDatabase };
 
@@ -113,8 +113,8 @@ export class FileDB {
 		const fileDB = new FileDB(client);
 
 		if (typeof window === "undefined") {
-			const database = (await import("jsr:@db/sqlite@0.11")).Database;
-			fileDB.db = { type: "SQLITE", db: new database("filemanager.db") };
+			const { Database } = await import("jsr:@db/sqlite");
+			fileDB.db = { type: "SQLITE", db: new Database("filemanager.db") };
 			fileDB.db.db.exec(`
 				CREATE TABLE IF NOT EXISTS file (
 					hash TEXT PRIMARY KEY,
@@ -396,7 +396,7 @@ class File implements FileAttributes {
 		}
 		if (!values.hash && values.id) {
 			console.log(`Fetching file metadata`); // TODO: Merge with getMetadata
-			const responses = await client.peers.fetch(`http://localhost/file/${values.id}`);
+			const responses = await client.rpcClient.fetch(`http://localhost/file/${values.id}`);
 			for (let i = 0; i < responses.length; i++) {
 				const response = await responses[i];
 				if (!response) continue;
@@ -438,7 +438,7 @@ class File implements FileAttributes {
 
 		const id = this.id;
 		if (id !== undefined && id !== null && id.length > 0) {
-			const responses = await this._client.peers.fetch(`http://localhost/file/${this.id}`);
+			const responses = await this._client.rpcClient.fetch(`http://localhost/file/${this.id}`);
 
 			for (let i = 0; i < responses.length; i++) {
 				try {
@@ -606,7 +606,7 @@ class File implements FileAttributes {
 			if (this._client.config.s3Endpoint.length > 0) file = await this.fetchFromS3();
 			if (file !== false) console.log(`  ${hash}  Serving ${this.size !== undefined ? Math.round(this.size / 1024 / 1024) : 0}MB from S3`);
 			else {
-				file = await this._client.peers.downloadFile(hash, this.size);
+				file = await this._client.rpcClient.downloadFile(hash, this.size);
 				if (file === false) {
 					this.found = false;
 					this.save();

src/hydrafiles.ts

@@ -1,14 +1,12 @@
-import Base32 from "npm:base32";
+import { encode as base32Encode } from "https://deno.land/[email protected]/encoding/base32.ts";
 // import WebTorrent from "npm:webtorrent";
 import getConfig, { type Config } from "./config.ts";
 import File, { type FileAttributes, FileDB } from "./file.ts";
-import startServer, { hashLocks } from "./server.ts";
 import Utils from "./utils.ts";
 // import Blockchain, { Block } from "./block.ts";
 import { S3Client } from "https://deno.land/x/[email protected]/mod.ts";
-import Peers from "./peers/peers.ts";
-import RTCPeers from "./peers/RTCPeers.ts";
-import HTTPPeers from "./peers/HTTPPeers.ts";
+import RPCServer from "./rpc/server.ts";
+import RPCClient from "./rpc/client.ts";
 import FileSystem from "./filesystem/filesystem.ts";
 
 // TODO: IDEA: HydraTorrent - New Github repo - "Hydrafiles + WebTorrent Compatibility Layer" - Hydrafiles noes can optionally run HydraTorrent to seed files via webtorrent
@@ -24,18 +22,15 @@ import FileSystem from "./filesystem/filesystem.ts";
 
 class Hydrafiles {
 	startTime: number = +new Date();
+	fs: FileSystem;
 	utils: Utils;
 	config: Config;
 	s3: S3Client | undefined;
-	// webtorrent: WebTorrent = new WebTorrent();
-	// blockchain = new Blockchain(this);
 	keyPair!: CryptoKeyPair;
+	rpcServer!: RPCServer;
+	rpcClient!: RPCClient;
 	fileDB!: FileDB;
-	peers!: Peers;
-	rtc!: RTCPeers;
-	http!: HTTPPeers;
-	fs: FileSystem;
-	handleRequest?: (req: Request) => Promise<string>;
+	// webtorrent: WebTorrent = new WebTorrent();
 
 	constructor(customConfig: Partial<Config> = {}) {
 		this.utils = new Utils(this);
@@ -48,30 +43,28 @@ class Hydrafiles {
 		}
 	}
 
-	public async start(onCompareFileListProgress?: (progress: number, total: number) => void): Promise<void> {
+	public async start(onUpdateFileListProgress?: (progress: number, total: number) => void): Promise<void> {
 		console.log("Startup: Populating KeyPair");
 		this.keyPair = await this.utils.getKeyPair();
 		console.log("Startup: Populating FileDB");
 		this.fileDB = await FileDB.init(this);
-		console.log("Startup: Populating Peers");
-		this.http = await HTTPPeers.init(this);
-		this.rtc = await RTCPeers.init(this);
-		this.peers = new Peers(this);
-
-		this.startBackgroundTasks(onCompareFileListProgress);
+		console.log("Startup: Populating RPC Client & Server");
+		this.rpcClient = new RPCClient(this);
+		this.rpcClient.start().then(() => {
+			this.rpcServer = new RPCServer(this);
+			this.startBackgroundTasks(onUpdateFileListProgress);
+		});
 	}
 
-	private startBackgroundTasks(onCompareFileListProgress?: (progress: number, total: number) => void): void {
-		startServer(this);
-
+	private startBackgroundTasks(onUpdateFileListProgress?: (progress: number, total: number) => void): void {
 		if (this.config.summarySpeed !== -1) setInterval(() => this.logState(), this.config.summarySpeed);
 		if (this.config.comparePeersSpeed !== -1) {
-			this.peers.fetchHTTPPeers();
-			setInterval(() => this.peers.fetchHTTPPeers(), this.config.comparePeersSpeed);
+			this.rpcClient.http.updatePeers();
+			setInterval(() => this.rpcClient.http.updatePeers(), this.config.comparePeersSpeed);
 		}
 		if (this.config.compareFilesSpeed !== -1) {
-			this.peers.compareFileList(onCompareFileListProgress);
-			setInterval(() => this.peers.compareFileList(onCompareFileListProgress), this.config.compareFilesSpeed);
+			this.updateFileList(onUpdateFileListProgress);
+			setInterval(() => this.updateFileList(onUpdateFileListProgress), this.config.compareFilesSpeed);
 		}
 		if (this.config.backfill) this.backfillFiles();
 	}
@@ -92,10 +85,62 @@ class Hydrafiles {
 		}
 	};
 
-	getHostname = async () => {
+	// TODO: Compare list between all peers and give score based on how similar they are. 100% = all exactly the same, 0% = no items in list were shared. The lower the score, the lower the propagation times, the lower the decentralisation
+	async updateFileList(onProgress?: (progress: number, total: number) => void): Promise<void> {
+		console.log(`Comparing file list`);
+		let files: FileAttributes[] = [];
+		const responses = await Promise.all(await this.rpcClient.fetch("http://localhost/files"));
+		for (let i = 0; i < responses.length; i++) {
+			if (responses[i] !== false) files = files.concat((await (responses[i] as Response).json()) as FileAttributes[]);
+		}
+
+		const uniqueFiles = new Set<string>();
+		files = files.filter((file) => {
+			const fileString = JSON.stringify(file);
+			if (!uniqueFiles.has(fileString)) {
+				uniqueFiles.add(fileString);
+				return true;
+			}
+			return false;
+		});
+
+		for (let i = 0; i < files.length; i++) {
+			if (onProgress) onProgress(i, files.length);
+			const newFile = files[i];
+			try {
+				if (typeof files[i].hash === "undefined") continue;
+				const fileObj: Partial<FileAttributes> = { hash: files[i].hash };
+				if (files[i].infohash) fileObj.infohash = files[i].infohash;
+				const currentFile = await File.init(fileObj, this);
+				if (!currentFile) continue;
+
+				const keys = Object.keys(newFile) as unknown as (keyof File)[];
+				for (let i = 0; i < keys.length; i++) {
+					const key = keys[i] as keyof FileAttributes;
+					if (["downloadCount", "voteHash", "voteNonce", "voteDifficulty"].includes(key)) continue;
+					if (newFile[key] !== undefined && newFile[key] !== null && newFile[key] !== 0 && (currentFile[key] === undefined || currentFile[key] === null || currentFile[key] === 0)) {
+						// @ts-expect-error:
+						currentFile[key] = newFile[key];
+					}
+					if (newFile.voteNonce !== 0 && newFile.voteDifficulty > currentFile.voteDifficulty) {
+						console.log(`  ${newFile.hash}  Checking vote nonce`);
+						currentFile.checkVoteNonce(newFile["voteNonce"]);
+					}
+				}
+				currentFile.save();
+			} catch (e) {
+				console.error(e);
+			}
+		}
+		if (onProgress) onProgress(files.length, files.length);
+	}
+
+	async getHostname(): Promise<string> {
 		const pubKey = await Utils.exportPublicKey(this.keyPair.publicKey);
-		return Base32.encode(pubKey.x).toLowerCase().replaceAll("=", "") + "." + Base32.encode(pubKey.y).toLowerCase().replaceAll("=", "");
-	};
+		const xEncoded = base32Encode(new TextEncoder().encode(pubKey.x)).toLowerCase().replace(/=+$/, "");
+		const yEncoded = base32Encode(new TextEncoder().encode(pubKey.y)).toLowerCase().replace(/=+$/, "");
+		return `${xEncoded}.${yEncoded}`;
+	}
 
 	async logState(): Promise<void> {
 		console.log(
@@ -113,9 +158,9 @@ class Hydrafiles {
 			(await this.fs.readDir("files/")).length,
 			`(${Math.round((100 * await this.utils.calculateUsedStorage()) / 1024 / 1024 / 1024) / 100}GB)`,
 			"\n| Processing Files:",
-			hashLocks.size,
-			"\n| Known Nodes:",
-			(await this.http.getPeers()).length,
+			this.rpcServer.hashLocks.size,
+			"\n| Known HTTP Peers:",
+			(await this.rpcClient.http.getPeers()).length,
 			// '\n| Seeding Torrent Files:',
 			// (await webtorrentClient()).torrents.length,
 			"\n| Downloads Served:",