diff --git a/build.ts b/build.ts index 2ea64ee..ff36fa7 100644 --- a/build.ts +++ b/build.ts @@ -1,6 +1,18 @@ import * as esbuild from "npm:esbuild"; import { denoPlugins } from "jsr:@luca/esbuild-deno-loader"; -console.log(await esbuild.build({ plugins: [...denoPlugins()], entryPoints: ["./src/hydrafiles.ts"], outfile: "./build/hydrafiles-web.esm.js", bundle: true, format: "esm", platform: "browser", sourcemap: true })); +console.log( + await esbuild.build({ + plugins: [...denoPlugins()], + entryPoints: ["./src/hydrafiles.ts"], + outfile: "./build/hydrafiles-web.esm.js", + bundle: true, + format: "esm", + platform: "browser", + sourcemap: true, + minify: true, + treeShaking: true, + }), +); esbuild.stop(); diff --git a/deno.jsonc b/deno.jsonc index caed12a..93bf347 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -1,6 +1,6 @@ { "name": "@starfiles/hydrafiles", - "version": "0.7.30", + "version": "0.7.31", "description": "The web privacy layer.", "main": "src/hydrafiles.ts", "exports": { diff --git a/deno.lock b/deno.lock index 50baa46..ed7418f 100644 --- a/deno.lock +++ b/deno.lock @@ -1,6 +1,7 @@ { "version": "4", "specifiers": { + "jsr:@db/sqlite@*": "0.12.0", "jsr:@db/sqlite@0.11": "0.11.1", "jsr:@denosaurs/plug@1": "1.0.6", "jsr:@luca/esbuild-deno-loader@*": "0.11.0", @@ -12,11 +13,11 @@ "jsr:@std/encoding@0.221": "0.221.0", "jsr:@std/encoding@^1.0.5": "1.0.5", "jsr:@std/fmt@0.221": "0.221.0", + "jsr:@std/fs@*": "1.0.4", "jsr:@std/fs@0.221": "0.221.0", "jsr:@std/path@0.217": "0.217.0", "jsr:@std/path@0.221": "0.221.0", "jsr:@std/path@^1.0.6": "1.0.6", - "npm:base32@*": "0.0.7", "npm:esbuild@*": "0.24.0", "npm:werift@*": "0.20.1" }, @@ -28,12 +29,19 @@ "jsr:@std/path@0.217" ] }, + "@db/sqlite@0.12.0": { + "integrity": "dd1ef7f621ad50fc1e073a1c3609c4470bd51edc0994139c5bf9851de7a6d85f", + "dependencies": [ + "jsr:@denosaurs/plug", + "jsr:@std/path@0.217" + ] + }, "@denosaurs/plug@1.0.6": { "integrity": "6cf5b9daba7799837b9ffbe89f3450510f588fafef8115ddab1ff0be9cb7c1a7", "dependencies": [ "jsr:@std/encoding@0.221", "jsr:@std/fmt", - "jsr:@std/fs", + "jsr:@std/fs@0.221", "jsr:@std/path@0.221" ] }, @@ -73,6 +81,9 @@ "jsr:@std/path@0.221" ] }, + "@std/fs@1.0.4": { + "integrity": "2907d32d8d1d9e540588fd5fe0ec21ee638134bd51df327ad4e443aaef07123c" + }, "@std/path@0.217.0": { "integrity": "1217cc25534bca9a2f672d7fe7c6f356e4027df400c0e85c0ef3e4343bc67d11", "dependencies": [ @@ -349,12 +360,6 @@ "tslib@2.7.0" ] }, - "base32@0.0.7": { - "integrity": "sha512-ire9Jmh+BsUk4Idu0wu6aKeJJr/2j28Mlu0qqJBd1SyOGxW/VRotkfwAv3/KE/entVlNwCefs9bxi7kBeCIxTQ==", - "dependencies": [ - "minimist" - ] - }, "base64-js@1.5.1": { "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==" }, @@ -506,9 +511,6 @@ "minimalistic-crypto-utils@1.0.1": { "integrity": "sha512-JIYlbt6g8i5jKfJ3xz7rF0LXmv2TkDxBLUkiBeZ7bAx4GnnNMr8xFpGnOxn6GhTEHx3SjRrZEoU+j04prX1ktg==" }, - "minimist@1.2.8": { - "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==" - }, "mp4box@0.5.2": { "integrity": "sha512-zRmGlvxy+YdW3Dmt+TR4xPHynbxwXtAQDTN/Fo9N3LMxaUlB2C5KmZpzYyGKy4c7k4Jf3RCR0A2pm9SZELOLXw==" }, @@ -672,9 +674,6 @@ ] } }, - "redirects": { - "https://deno.land/std/path/mod.ts": "https://deno.land/std@0.224.0/path/mod.ts" - }, "remote": { "https://deno.land/std@0.170.0/_util/asserts.ts": "d0844e9b62510f89ce1f9878b046f6a57bf88f208a10304aab50efcb48365272", "https://deno.land/std@0.170.0/_util/os.ts": "8a33345f74990e627b9dfe2de9b040004b08ea5146c7c9e8fe9a29070d193934", @@ -799,6 +798,7 @@ "https://deno.land/std@0.170.0/path/separator.ts": "fe1816cb765a8068afb3e8f13ad272351c85cbc739af56dacfc7d93d710fe0f9", "https://deno.land/std@0.170.0/path/win32.ts": "7cebd2bda6657371adc00061a1d23fdd87bcdf64b4843bb148b0b24c11b40f69", "https://deno.land/std@0.170.0/streams/write_all.ts": "7525aa90e34a2bb56d094403ad9c0e31e7db87a47cb556e6dc0404e6deca084c", + "https://deno.land/std@0.194.0/encoding/base32.ts": "c329447451560ec692b9eb4d1badb6437f1d419ddbb21c1f994b0fe0b6b66cc8", "https://deno.land/std@0.215.0/assert/assert.ts": "bec068b2fccdd434c138a555b19a2c2393b71dfaada02b7d568a01541e67cdc5", "https://deno.land/std@0.215.0/assert/assertion_error.ts": "9f689a101ee586c4ce92f52fa7ddd362e86434ffdf1f848e45987dc7689976b8", "https://deno.land/std@0.215.0/bytes/copy.ts": "f29c03168853720dfe82eaa57793d0b9e3543ebfe5306684182f0f1e3bfd422a", diff --git a/src/file.ts b/src/file.ts index a8147f1..183a794 100644 --- a/src/file.ts +++ b/src/file.ts @@ -2,7 +2,7 @@ import type Hydrafiles from "./hydrafiles.ts"; import Utils, { type NonNegativeNumber } from "./utils.ts"; import type { indexedDB } from "https://deno.land/x/indexeddb@v1.1.0/ponyfill.ts"; import { join } from "https://deno.land/std@0.224.0/path/mod.ts"; -import type { Database } from "jsr:@db/sqlite@0.11"; +import type { Database } from "jsr:@db/sqlite"; type DatabaseWrapper = { type: "UNDEFINED"; db: undefined } | { type: "SQLITE"; db: Database } | { type: "INDEXEDDB"; db: IDBDatabase }; @@ -113,8 +113,8 @@ export class FileDB { const fileDB = new FileDB(client); if (typeof window === "undefined") { - const database = (await import("jsr:@db/sqlite@0.11")).Database; - fileDB.db = { type: "SQLITE", db: new database("filemanager.db") }; + const { Database } = await import("jsr:@db/sqlite"); + fileDB.db = { type: "SQLITE", db: new Database("filemanager.db") }; fileDB.db.db.exec(` CREATE TABLE IF NOT EXISTS file ( hash TEXT PRIMARY KEY, @@ -396,7 +396,7 @@ class File implements FileAttributes { } if (!values.hash && values.id) { console.log(`Fetching file metadata`); // TODO: Merge with getMetadata - const responses = await client.peers.fetch(`http://localhost/file/${values.id}`); + const responses = await client.rpcClient.fetch(`http://localhost/file/${values.id}`); for (let i = 0; i < responses.length; i++) { const response = await responses[i]; if (!response) continue; @@ -438,7 +438,7 @@ class File implements FileAttributes { const id = this.id; if (id !== undefined && id !== null && id.length > 0) { - const responses = await this._client.peers.fetch(`http://localhost/file/${this.id}`); + const responses = await this._client.rpcClient.fetch(`http://localhost/file/${this.id}`); for (let i = 0; i < responses.length; i++) { try { @@ -606,7 +606,7 @@ class File implements FileAttributes { if (this._client.config.s3Endpoint.length > 0) file = await this.fetchFromS3(); if (file !== false) console.log(` ${hash} Serving ${this.size !== undefined ? Math.round(this.size / 1024 / 1024) : 0}MB from S3`); else { - file = await this._client.peers.downloadFile(hash, this.size); + file = await this._client.rpcClient.downloadFile(hash, this.size); if (file === false) { this.found = false; this.save(); diff --git a/src/hydrafiles.ts b/src/hydrafiles.ts index d087d16..ea9a236 100644 --- a/src/hydrafiles.ts +++ b/src/hydrafiles.ts @@ -1,14 +1,12 @@ -import Base32 from "npm:base32"; +import { encode as base32Encode } from "https://deno.land/std@0.194.0/encoding/base32.ts"; // import WebTorrent from "npm:webtorrent"; import getConfig, { type Config } from "./config.ts"; import File, { type FileAttributes, FileDB } from "./file.ts"; -import startServer, { hashLocks } from "./server.ts"; import Utils from "./utils.ts"; // import Blockchain, { Block } from "./block.ts"; import { S3Client } from "https://deno.land/x/s3_lite_client@0.7.0/mod.ts"; -import Peers from "./peers/peers.ts"; -import RTCPeers from "./peers/RTCPeers.ts"; -import HTTPPeers from "./peers/HTTPPeers.ts"; +import RPCServer from "./rpc/server.ts"; +import RPCClient from "./rpc/client.ts"; import FileSystem from "./filesystem/filesystem.ts"; // TODO: IDEA: HydraTorrent - New Github repo - "Hydrafiles + WebTorrent Compatibility Layer" - Hydrafiles noes can optionally run HydraTorrent to seed files via webtorrent @@ -24,18 +22,15 @@ import FileSystem from "./filesystem/filesystem.ts"; class Hydrafiles { startTime: number = +new Date(); + fs: FileSystem; utils: Utils; config: Config; s3: S3Client | undefined; - // webtorrent: WebTorrent = new WebTorrent(); - // blockchain = new Blockchain(this); keyPair!: CryptoKeyPair; + rpcServer!: RPCServer; + rpcClient!: RPCClient; fileDB!: FileDB; - peers!: Peers; - rtc!: RTCPeers; - http!: HTTPPeers; - fs: FileSystem; - handleRequest?: (req: Request) => Promise; + // webtorrent: WebTorrent = new WebTorrent(); constructor(customConfig: Partial = {}) { this.utils = new Utils(this); @@ -48,30 +43,28 @@ class Hydrafiles { } } - public async start(onCompareFileListProgress?: (progress: number, total: number) => void): Promise { + public async start(onUpdateFileListProgress?: (progress: number, total: number) => void): Promise { console.log("Startup: Populating KeyPair"); this.keyPair = await this.utils.getKeyPair(); console.log("Startup: Populating FileDB"); this.fileDB = await FileDB.init(this); - console.log("Startup: Populating Peers"); - this.http = await HTTPPeers.init(this); - this.rtc = await RTCPeers.init(this); - this.peers = new Peers(this); - - this.startBackgroundTasks(onCompareFileListProgress); + console.log("Startup: Populating RPC Client & Server"); + this.rpcClient = new RPCClient(this); + this.rpcClient.start().then(() => { + this.rpcServer = new RPCServer(this); + this.startBackgroundTasks(onUpdateFileListProgress); + }); } - private startBackgroundTasks(onCompareFileListProgress?: (progress: number, total: number) => void): void { - startServer(this); - + private startBackgroundTasks(onUpdateFileListProgress?: (progress: number, total: number) => void): void { if (this.config.summarySpeed !== -1) setInterval(() => this.logState(), this.config.summarySpeed); if (this.config.comparePeersSpeed !== -1) { - this.peers.fetchHTTPPeers(); - setInterval(() => this.peers.fetchHTTPPeers(), this.config.comparePeersSpeed); + this.rpcClient.http.updatePeers(); + setInterval(() => this.rpcClient.http.updatePeers(), this.config.comparePeersSpeed); } if (this.config.compareFilesSpeed !== -1) { - this.peers.compareFileList(onCompareFileListProgress); - setInterval(() => this.peers.compareFileList(onCompareFileListProgress), this.config.compareFilesSpeed); + this.updateFileList(onUpdateFileListProgress); + setInterval(() => this.updateFileList(onUpdateFileListProgress), this.config.compareFilesSpeed); } if (this.config.backfill) this.backfillFiles(); } @@ -92,10 +85,62 @@ class Hydrafiles { } }; - getHostname = async () => { + // TODO: Compare list between all peers and give score based on how similar they are. 100% = all exactly the same, 0% = no items in list were shared. The lower the score, the lower the propagation times, the lower the decentralisation + async updateFileList(onProgress?: (progress: number, total: number) => void): Promise { + console.log(`Comparing file list`); + let files: FileAttributes[] = []; + const responses = await Promise.all(await this.rpcClient.fetch("http://localhost/files")); + for (let i = 0; i < responses.length; i++) { + if (responses[i] !== false) files = files.concat((await (responses[i] as Response).json()) as FileAttributes[]); + } + + const uniqueFiles = new Set(); + files = files.filter((file) => { + const fileString = JSON.stringify(file); + if (!uniqueFiles.has(fileString)) { + uniqueFiles.add(fileString); + return true; + } + return false; + }); + + for (let i = 0; i < files.length; i++) { + if (onProgress) onProgress(i, files.length); + const newFile = files[i]; + try { + if (typeof files[i].hash === "undefined") continue; + const fileObj: Partial = { hash: files[i].hash }; + if (files[i].infohash) fileObj.infohash = files[i].infohash; + const currentFile = await File.init(fileObj, this); + if (!currentFile) continue; + + const keys = Object.keys(newFile) as unknown as (keyof File)[]; + for (let i = 0; i < keys.length; i++) { + const key = keys[i] as keyof FileAttributes; + if (["downloadCount", "voteHash", "voteNonce", "voteDifficulty"].includes(key)) continue; + if (newFile[key] !== undefined && newFile[key] !== null && newFile[key] !== 0 && (currentFile[key] === undefined || currentFile[key] === null || currentFile[key] === 0)) { + // @ts-expect-error: + currentFile[key] = newFile[key]; + } + if (newFile.voteNonce !== 0 && newFile.voteDifficulty > currentFile.voteDifficulty) { + console.log(` ${newFile.hash} Checking vote nonce`); + currentFile.checkVoteNonce(newFile["voteNonce"]); + } + } + currentFile.save(); + } catch (e) { + console.error(e); + } + } + if (onProgress) onProgress(files.length, files.length); + } + + async getHostname(): Promise { const pubKey = await Utils.exportPublicKey(this.keyPair.publicKey); - return Base32.encode(pubKey.x).toLowerCase().replaceAll("=", "") + "." + Base32.encode(pubKey.y).toLowerCase().replaceAll("=", ""); - }; + const xEncoded = base32Encode(new TextEncoder().encode(pubKey.x)).toLowerCase().replace(/=+$/, ""); + const yEncoded = base32Encode(new TextEncoder().encode(pubKey.y)).toLowerCase().replace(/=+$/, ""); + return `${xEncoded}.${yEncoded}`; + } async logState(): Promise { console.log( @@ -113,9 +158,9 @@ class Hydrafiles { (await this.fs.readDir("files/")).length, `(${Math.round((100 * await this.utils.calculateUsedStorage()) / 1024 / 1024 / 1024) / 100}GB)`, "\n| Processing Files:", - hashLocks.size, - "\n| Known Nodes:", - (await this.http.getPeers()).length, + this.rpcServer.hashLocks.size, + "\n| Known HTTP Peers:", + (await this.rpcClient.http.getPeers()).length, // '\n| Seeding Torrent Files:', // (await webtorrentClient()).torrents.length, "\n| Downloads Served:", diff --git a/src/peers/peers.ts b/src/peers/peers.ts deleted file mode 100644 index a546ba4..0000000 --- a/src/peers/peers.ts +++ /dev/null @@ -1,137 +0,0 @@ -import type { FileAttributes } from "../file.ts"; -import type Hydrafiles from "../hydrafiles.ts"; -import { HTTPPeer } from "./HTTPPeers.ts"; -import File from "../file.ts"; -import Utils from "../utils.ts"; - -export default class Peers { - _client: Hydrafiles; - - constructor(client: Hydrafiles) { - this._client = client; - } - - public async fetch(input: RequestInfo, init?: RequestInit): Promise[]> { - return [...await this._client.http.fetch(input, init), ...this._client.rtc.fetch(input, init)]; - } - - async announceHTTP(): Promise { - await Promise.all([...await this._client.http.fetch(`http://localhost/announce?host=${this._client.config.publicHostname}`), ...this._client.rtc.fetch(`http://localhost/announce?host=${this._client.config.publicHostname}`)]); - } - - // TODO: Compare list between all peers and give score based on how similar they are. 100% = all exactly the same, 0% = no items in list were shared. The lower the score, the lower the propagation times, the lower the decentralisation - async fetchHTTPPeers(): Promise { - console.log(`Fetching peers`); - const responses = await Promise.all(await this.fetch("http://localhost/peers")); - for (let i = 0; i < responses.length; i++) { - try { - if (!(responses[i] instanceof Response)) continue; - const response = responses[i]; - if (response instanceof Response) { - const remotePeers = (await response.json()) as HTTPPeer[]; - for (const remotePeer of remotePeers) { - this._client.http.add(remotePeer.host).catch((e) => { - if (this._client.config.logLevel === "verbose") console.error(e); - }); - } - } - } catch (e) { - if (this._client.config.logLevel === "verbose") console.error(e); - } - } - } - - // TODO: Compare list between all peers and give score based on how similar they are. 100% = all exactly the same, 0% = no items in list were shared. The lower the score, the lower the propagation times, the lower the decentralisation - async compareFileList(onProgress?: (progress: number, total: number) => void): Promise { - console.log(`Comparing file list`); - let files: FileAttributes[] = []; - const responses = await Promise.all(await this.fetch("http://localhost/files")); - for (let i = 0; i < responses.length; i++) { - if (responses[i] !== false) files = files.concat((await (responses[i] as Response).json()) as FileAttributes[]); - } - - const uniqueFiles = new Set(); - files = files.filter((file) => { - const fileString = JSON.stringify(file); - if (!uniqueFiles.has(fileString)) { - uniqueFiles.add(fileString); - return true; - } - return false; - }); - - for (let i = 0; i < files.length; i++) { - if (onProgress) onProgress(i, files.length); - const newFile = files[i]; - try { - if (typeof files[i].hash === "undefined") continue; - const fileObj: Partial = { hash: files[i].hash }; - if (files[i].infohash) fileObj.infohash = files[i].infohash; - const currentFile = await File.init(fileObj, this._client); - if (!currentFile) continue; - - const keys = Object.keys(newFile) as unknown as (keyof File)[]; - for (let i = 0; i < keys.length; i++) { - const key = keys[i] as keyof FileAttributes; - if (["downloadCount", "voteHash", "voteNonce", "voteDifficulty"].includes(key)) continue; - if (newFile[key] !== undefined && newFile[key] !== null && newFile[key] !== 0 && (currentFile[key] === undefined || currentFile[key] === null || currentFile[key] === 0)) { - // @ts-expect-error: - currentFile[key] = newFile[key]; - } - if (newFile.voteNonce !== 0 && newFile.voteDifficulty > currentFile.voteDifficulty) { - console.log(` ${newFile.hash} Checking vote nonce`); - currentFile.checkVoteNonce(newFile["voteNonce"]); - } - } - currentFile.save(); - } catch (e) { - console.error(e); - } - } - if (onProgress) onProgress(files.length, files.length); - } - - async downloadFile(hash: string, size = 0): Promise<{ file: Uint8Array; signal: number } | false> { - if (!this._client.utils.hasSufficientMemory(size)) { - console.log("Reached memory limit, waiting"); - await new Promise(() => { - const intervalId = setInterval(async () => { - if (await this._client.utils.hasSufficientMemory(size)) clearInterval(intervalId); - }, this._client.config.memoryThresholdReachedWait); - }); - } - - const file = await File.init({ hash }, this._client); - if (!file) return false; - const peers = await this._client.http.getPeers(true); - for (const peer of peers) { - let fileContent: { file: Uint8Array; signal: number } | false = false; - try { - fileContent = await this._client.http.downloadFromPeer(await HTTPPeer.init(peer, this._client.http._db), file); - } catch (e) { - console.error(e); - } - if (fileContent) return fileContent; - } - - console.log(` ${hash} Downloading from WebRTC`); - const responses = this._client.rtc.fetch(`http://localhost/download/${hash}`); - for (let i = 0; i < responses.length; i++) { - const hash = file.hash; - const response = await responses[i]; - const peerContent = new Uint8Array(await response.arrayBuffer()); - console.log(` ${hash} Validating hash`); - const verifiedHash = await Utils.hashUint8Array(peerContent); - console.log(` ${hash} Done Validating hash`); - if (hash !== verifiedHash) return false; - console.log(` ${hash} Valid hash`); - - if (file.name === undefined || file.name === null || file.name.length === 0) { - file.name = String(response.headers.get("Content-Disposition")?.split("=")[1].replace(/"/g, "").replace(" [HYDRAFILES]", "")); - file.save(); - } - } - - return false; - } -} diff --git a/src/rpc/client.ts b/src/rpc/client.ts new file mode 100644 index 0000000..666bbc5 --- /dev/null +++ b/src/rpc/client.ts @@ -0,0 +1,67 @@ +import type Hydrafiles from "../hydrafiles.ts"; +import HTTPClient, { HTTPPeer } from "./peers/http.ts"; +import RTCClient from "./peers/rtc.ts"; +import File from "../file.ts"; +import Utils from "../utils.ts"; + +export default class RPCClient { + _client: Hydrafiles; + http!: HTTPClient; + rtc!: RTCClient; + + constructor(client: Hydrafiles) { + this._client = client; + } + async start(): Promise { + this.http = await HTTPClient.init(this._client); + this.rtc = await RTCClient.init(this._client); + } + + public async fetch(input: RequestInfo, init?: RequestInit): Promise[]> { + return [...await this.http.fetch(input, init), ...this.rtc.fetch(input, init)]; + } + + async downloadFile(hash: string, size = 0): Promise<{ file: Uint8Array; signal: number } | false> { + if (!this._client.utils.hasSufficientMemory(size)) { + console.log("Reached memory limit, waiting"); + await new Promise(() => { + const intervalId = setInterval(async () => { + if (await this._client.utils.hasSufficientMemory(size)) clearInterval(intervalId); + }, this._client.config.memoryThresholdReachedWait); + }); + } + + const file = await File.init({ hash }, this._client); + if (!file) return false; + const peers = await this.http.getPeers(true); + for (const peer of peers) { + let fileContent: { file: Uint8Array; signal: number } | false = false; + try { + fileContent = await this.http.downloadFromPeer(await HTTPPeer.init(peer, this.http._db), file); + } catch (e) { + console.error(e); + } + if (fileContent) return fileContent; + } + + console.log(` ${hash} Downloading from WebRTC`); + const responses = this.rtc.fetch(`http://localhost/download/${hash}`); + for (let i = 0; i < responses.length; i++) { + const hash = file.hash; + const response = await responses[i]; + const peerContent = new Uint8Array(await response.arrayBuffer()); + console.log(` ${hash} Validating hash`); + const verifiedHash = await Utils.hashUint8Array(peerContent); + console.log(` ${hash} Done Validating hash`); + if (hash !== verifiedHash) return false; + console.log(` ${hash} Valid hash`); + + if (file.name === undefined || file.name === null || file.name.length === 0) { + file.name = String(response.headers.get("Content-Disposition")?.split("=")[1].replace(/"/g, "").replace(" [HYDRAFILES]", "")); + file.save(); + } + } + + return false; + } +} diff --git a/src/peers/HTTPPeers.ts b/src/rpc/peers/http.ts similarity index 93% rename from src/peers/HTTPPeers.ts rename to src/rpc/peers/http.ts index 6c98ee0..b06d879 100644 --- a/src/peers/HTTPPeers.ts +++ b/src/rpc/peers/http.ts @@ -1,8 +1,8 @@ -import type Hydrafiles from "../hydrafiles.ts"; -import Utils from "../utils.ts"; +import type Hydrafiles from "../../hydrafiles.ts"; +import Utils from "../../utils.ts"; import type { Database } from "jsr:@db/sqlite@0.11"; import type { indexedDB } from "https://deno.land/x/indexeddb@v1.1.0/ponyfill.ts"; -import File from "../file.ts"; +import File from "../../file.ts"; type DatabaseWrapper = { type: "UNDEFINED"; db: undefined } | { type: "SQLITE"; db: Database } | { type: "INDEXEDDB"; db: IDBDatabase }; @@ -356,7 +356,7 @@ export class HTTPPeer implements PeerAttributes { } // TODO: Log common user-agents and re-use them to help anonimise non Hydrafiles peers -export default class HTTPPeers { +export default class HTTPClient { private _client: Hydrafiles; public _db: PeerDB; @@ -365,9 +365,9 @@ export default class HTTPPeers { this._db = db; } - public static async init(client: Hydrafiles): Promise { + public static async init(client: Hydrafiles): Promise { const db = await PeerDB.init(client); - const peers = new HTTPPeers(client, db); + const peers = new HTTPClient(client, db); for (let i = 0; i < client.config.bootstrapPeers.length; i++) { await peers.add(client.config.bootstrapPeers[i]); @@ -483,4 +483,26 @@ export default class HTTPPeers { return fetchPromises; } + + // TODO: Compare list between all peers and give score based on how similar they are. 100% = all exactly the same, 0% = no items in list were shared. The lower the score, the lower the propagation times, the lower the decentralisation + async updatePeers(): Promise { + console.log(`Fetching peers`); + const responses = await Promise.all(await this._client.rpcClient.fetch("http://localhost/peers")); + for (let i = 0; i < responses.length; i++) { + try { + if (!(responses[i] instanceof Response)) continue; + const response = responses[i]; + if (response instanceof Response) { + const remotePeers = (await response.json()) as HTTPPeer[]; + for (const remotePeer of remotePeers) { + this.add(remotePeer.host).catch((e) => { + if (this._client.config.logLevel === "verbose") console.error(e); + }); + } + } + } catch (e) { + if (this._client.config.logLevel === "verbose") console.error(e); + } + } + } } diff --git a/src/peers/RTCPeers.ts b/src/rpc/peers/rtc.ts similarity index 97% rename from src/peers/RTCPeers.ts rename to src/rpc/peers/rtc.ts index c87fbc8..7b19cbd 100644 --- a/src/peers/RTCPeers.ts +++ b/src/rpc/peers/rtc.ts @@ -1,6 +1,5 @@ import type { RTCDataChannel, RTCIceCandidate, RTCPeerConnection, RTCSessionDescription } from "npm:werift"; -import { handleRequest } from "../server.ts"; -import type Hydrafiles from "../hydrafiles.ts"; +import type Hydrafiles from "../../hydrafiles.ts"; function extractIPAddress(sdp: string): string { const ipv4Regex = /c=IN IP4 (\d{1,3}(?:\.\d{1,3}){3})/g; @@ -36,7 +35,7 @@ function arrayBufferToUnicodeString(buffer: ArrayBuffer): string { const peerId = Math.random(); -class RTCPeers { +class RTCClient { _client: Hydrafiles; peerId: number; websockets: WebSocket[]; @@ -51,9 +50,9 @@ class RTCPeers { this.seenMessages = new Set(); } - static async init(client: Hydrafiles): Promise { - const webRTC = new RTCPeers(client); - const peers = await client.http.getPeers(true); + static async init(client: Hydrafiles): Promise { + const webRTC = new RTCClient(client); + const peers = await client.rpcClient.http.getPeers(true); for (let i = 0; i < peers.length; i++) { try { webRTC.websockets.push(new WebSocket(peers[i].host.replace("https://", "wss://").replace("http://", "ws://"))); @@ -106,7 +105,7 @@ class RTCPeers { console.log(`WebRTC: (10/12): Received request`); const { id, url, ...data } = JSON.parse(e.data as string); const req = new Request(url, data); - const response = await handleRequest(req, this._client); + const response = await this._client.rpcServer.handleRequest(req); const headersObj: Record = {}; response.headers.forEach((value, key) => { headersObj[key] = value; @@ -296,4 +295,4 @@ class RTCPeers { } } -export default RTCPeers; +export default RTCClient; diff --git a/src/rpc/server.ts b/src/rpc/server.ts new file mode 100644 index 0000000..5a2dc9f --- /dev/null +++ b/src/rpc/server.ts @@ -0,0 +1,379 @@ +import { encode as base32Encode } from "https://deno.land/std@0.194.0/encoding/base32.ts"; +import type Hydrafiles from "../hydrafiles.ts"; +// import { BLOCKSDIR } from "./block.ts"; +import File, { fileAttributesDefaults } from "../file.ts"; +import Utils from "../utils.ts"; +import { join } from "https://deno.land/std@0.224.0/path/mod.ts"; +import type Base64 from "npm:base64"; +import { HTTPPeer } from "./peers/http.ts"; +import { Message } from "./peers/rtc.ts"; + +class RPCServer { + private _client: Hydrafiles; + private cachedHostnames: { [key: string]: { body: string; headers: Headers } } = {}; + private sockets: { id: number; socket: WebSocket }[] = []; + public hashLocks = new Map>(); + public handleCustomRequest?: (req: Request) => Promise; + + constructor(client: Hydrafiles) { + this._client = client; + + if (typeof window !== "undefined") return; + let port = client.config.port; + while (true) { + try { + Deno.serve({ + port, + hostname: client.config.hostname, + onListen: ({ hostname, port }): void => { + this.onListen(hostname, port); + }, + handler: async (req: Request): Promise => await this.handleRequest(req), + }); + return; + } catch (e) { + const err = e as Error; + if (err.name !== "AddrInUse") throw err; + port++; + } + } + } + + onListen = async (hostname: string, port: number): Promise => { + console.log(`Server started at ${hostname}:${port}`); + console.log("Testing network connection"); + const file = await this._client.rpcClient.downloadFile("04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f"); + if (file === false) console.error("Download test failed, cannot connect to network"); + else { + console.log("Connected to network"); + if (Utils.isIp(this._client.config.publicHostname) && Utils.isPrivateIP(this._client.config.publicHostname)) console.error("Public hostname is a private IP address, cannot announce to other nodes"); + else { + console.log(`Testing downloads ${this._client.config.publicHostname}/download/04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f`); + const file = await File.init({ hash: "04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f" }, this._client); + if (!file) console.error("Failed to build file"); + else { + const response = await this._client.rpcClient.http.downloadFromPeer(await HTTPPeer.init({ host: this._client.config.publicHostname }, this._client.rpcClient.http._db), file); + if (response === false) console.error("Test: Failed to download file from self"); + else { + console.log("Announcing HTTP server to nodes"); + this._client.rpcClient.fetch(`http://localhost/announce?host=${this._client.config.publicHostname}`); + } + await this._client.rpcClient.http.add(this._client.config.publicHostname); + } + } + } + }; + + handleRequest = async (req: Request): Promise => { + console.log(`Received Request: ${req.url}`); + const url = new URL(req.url); + const headers = new Headers(); + headers.set("Access-Control-Allow-Origin", "*"); + + try { + if (req.headers.get("upgrade") === "websocket") { + const { socket, response } = Deno.upgradeWebSocket(req); + this.sockets.push({ socket, id: 0 }); + + socket.addEventListener("message", ({ data }) => { + const message = JSON.parse(data) as Message | null; + if (message === null) return; + for (let i = 0; i < this.sockets.length; i++) { + if (this.sockets[i].socket !== socket && (!("to" in message) || message.to === this.sockets[i].id)) { + if (this.sockets[i].socket.readyState === 1) this.sockets[i].socket.send(data); + } else if ("from" in message) { + this.sockets[i].id = message.from; + } + } + }); + + return response; + } else if (url.pathname === "/" || url.pathname === undefined) { + headers.set("Content-Type", "text/html"); + headers.set("Cache-Control", "public, max-age=604800"); + return new Response(await this._client.fs.readFile("public/index.html") || "", { headers }); + } else if (url.pathname === "/favicon.ico") { + headers.set("Content-Type", "image/x-icon"); + headers.set("Cache-Control", "public, max-age=604800"); + return new Response(await this._client.fs.readFile("public/favicon.ico") || "", { headers }); + } else if (url.pathname === "/status") { + headers.set("Content-Type", "application/json"); + return new Response(JSON.stringify({ status: true }), { headers }); + } else if (url.pathname === "/hydrafiles-web.esm.js") { + headers.set("Content-Type", "application/javascript"); + headers.set("Cache-Control", "public, max-age=300"); + return new Response(await this._client.fs.readFile("build/hydrafiles-web.esm.js") || "", { headers }); + } else if (url.pathname === "/hydrafiles-web.esm.js.map") { + headers.set("Content-Type", "application/json"); + headers.set("Cache-Control", "public, max-age=300"); + return new Response(await this._client.fs.readFile("build/hydrafiles-web.esm.js.map") || "", { headers }); + } else if (url.pathname === "/demo.html") { + headers.set("Content-Type", "text/html"); + headers.set("Cache-Control", "public, max-age=300"); + return new Response(await this._client.fs.readFile("public/demo.html") || "", { headers }); + } else if (url.pathname === "/dashboard.html") { + headers.set("Content-Type", "text/html"); + headers.set("Cache-Control", "public, max-age=300"); + return new Response(await this._client.fs.readFile("public/dashboard.html") || "", { headers }); + } else if (url.pathname === "/peers") { + headers.set("Content-Type", "application/json"); + headers.set("Cache-Control", "public, max-age=300"); + return new Response(JSON.stringify(await this._client.rpcClient.http.getPeers()), { headers }); + } else if (url.pathname === "/info") { + headers.set("Content-Type", "application/json"); + headers.set("Cache-Control", "public, max-age=300"); + return new Response(JSON.stringify({ version: JSON.parse(await Deno.readTextFile("deno.jsonc")).version }), { headers }); + } else if (url.pathname.startsWith("/announce")) { + const host = url.searchParams.get("host"); + if (host === null) return new Response("No hosted given\n", { status: 401 }); + const knownNodes = await this._client.rpcClient.http.getPeers(); + if (knownNodes.find((node) => node.host === host) !== undefined) return new Response("Already known\n"); + await this._client.rpcClient.http.add(host); + return new Response("Announced\n"); + } else if (url.pathname?.startsWith("/download/")) { + const hash = url.pathname.split("/")[2]; + const fileId = url.pathname.split("/")[3] ?? ""; + const infohash = Array.from(decodeURIComponent(url.searchParams.get("info_hash") ?? "")).map((char) => char.charCodeAt(0).toString(16).padStart(2, "0")).join(""); + + if (this.hashLocks.has(hash)) { + if (this._client.config.logLevel === "verbose") console.log(` ${hash} Waiting for existing request with same hash`); + await this.hashLocks.get(hash); + } + const processingPromise = (async () => { + const file = await File.init({ hash, infohash }, this._client, true); + if (!file) throw new Error("Failed to build file"); + + if (fileId.length !== 0) { + const id = file.id; + if (id === undefined || id === null || id.length === 0) { + file.id = fileId; + file.save(); + } + } + + await file.getMetadata(); + let fileContent: { file: Uint8Array; signal: number } | false; + try { + fileContent = await file.getFile({ logDownloads: true }); + } catch (e) { + const err = e as { message: string }; + if (err.message === "Promise timed out") { + fileContent = false; + } else { + throw e; + } + } + + if (fileContent === false) { + file.found = false; + file.save(); + return new Response("404 File Not Found\n", { + status: 404, + }); + } + + headers.set("Content-Type", "application/octet-stream"); + headers.set("Cache-Control", "public, max-age=31536000"); + headers.set("Content-Length", fileContent.file.byteLength.toString()); + headers.set("Signal-Strength", String(fileContent.signal)); + console.log(` ${hash} Signal Strength:`, fileContent.signal, Utils.estimateHops(fileContent.signal)); + + headers.set("Content-Length", String(file.size)); + if (file.name !== undefined && file.name !== null) { + headers.set("Content-Disposition", `attachment; filename="${encodeURIComponent(file.name.replace(/[^a-zA-Z0-9._-]/g, "").replace(/\s+/g, " ").trim()).replace(/%20/g, " ").replace(/(\.\w+)$/, " [HYDRAFILES]$1")}"`); + } + + return new Response(fileContent.file, { headers }); + })(); + + this.hashLocks.set(hash, processingPromise); + + let response: Response; + try { + response = await processingPromise; + } finally { + this.hashLocks.delete(hash); + } + return response; + } else if (url.pathname?.startsWith("/infohash/")) { + const infohash = url.pathname.split("/")[2]; + + if (this.hashLocks.has(infohash)) { + console.log(` ${infohash} Waiting for existing request with same infohash`); + await this.hashLocks.get(infohash); + } + const processingPromise = (async () => { + const file = await File.init({ infohash }, this._client, true); + if (!file) throw new Error("Failed to build file"); + + await file.getMetadata(); + let fileContent: { file: Uint8Array; signal: number } | false; + try { + fileContent = await file.getFile({ logDownloads: true }); + } catch (e) { + const err = e as { message: string }; + if (err.message === "Promise timed out") { + fileContent = false; + } else { + throw e; + } + } + + if (fileContent === false) { + file.found = false; + file.save(); + return new Response("404 File Not Found\n", { + status: 404, + }); + } + + headers.set("Content-Type", "application/octet-stream"); + headers.set("Cache-Control", "public, max-age=31536000"); + + headers.set("Signal-Strength", String(fileContent.signal)); + console.log(` ${file.hash} Signal Strength:`, fileContent.signal, Utils.estimateHops(fileContent.signal)); + + headers.set("Content-Length", String(file.size)); + if (file.name !== null) headers.set("Content-Disposition", `attachment; filename="${encodeURIComponent(file.name).replace(/%20/g, " ").replace(/(\.\w+)$/, " [HYDRAFILES]$1")}"`); + + return new Response(fileContent.file, { headers }); + })(); + + this.hashLocks.set(infohash, processingPromise); + + try { + await processingPromise; + } finally { + this.hashLocks.delete(infohash); + } + } else if (url.pathname === "/upload") { + const uploadSecret = req.headers.get("x-hydra-upload-secret"); + if (uploadSecret !== this._client.config.uploadSecret) { + return new Response("401 Unauthorized\n", { status: 401 }); + } + + const form = await req.formData(); + const formData = { + hash: form.get("hash")?.toString(), + file: form.get("file") as globalThis.File | null, + }; + + if (typeof formData.hash === "undefined" || typeof formData.file === "undefined" || formData.file === null) return new Response("400 Bad Request\n", { status: 400 }); + + const hash = formData.hash[0]; + + const file = await File.init({ hash }, this._client, true); + if (!file) throw new Error("Failed to build file"); + if ((file.name === null || file.name.length === 0) && formData.file.name !== null) { + file.name = formData.file.name; + file.cacheFile(new Uint8Array(await formData.file.arrayBuffer())); + file.save(); + } + + console.log("Uploading", file.hash); + + if (await this._client.fs.exists(join("files", file.hash))) return new Response("200 OK\n"); + + if (!this._client.config.permaFiles.includes(hash)) this._client.config.permaFiles.push(hash); + await this._client.fs.writeFile("config.json", new TextEncoder().encode(JSON.stringify(this._client.config, null, 2))); + return new Response("200 OK\n"); + } else if (url.pathname === "/files") { + const rows = (await this._client.fileDB.select()).map((row) => { + const { downloadCount, found, ...rest } = row; + const _ = { downloadCount, found }; + return rest; + }); + headers.set("Content-Type", "application/json"); + headers.set("Cache-Control", "public, max-age=10800"); + return new Response(JSON.stringify(rows), { headers }); + } else if (url.pathname.startsWith("/file/")) { + const id = url.pathname.split("/")[2]; + let file: File | false; + try { + file = await File.init({ id }, this._client, true); + } catch (e) { + const err = e as Error; + if (err.message === "File not found") return new Response("File not found", { headers, status: 404 }); + else throw err; + } + + headers.set("Content-Type", "application/json"); + headers.set("Cache-Control", "public, max-age=10800"); + if (!file) return new Response("File not found", { headers, status: 404 }); + return new Response(JSON.stringify(fileAttributesDefaults(file)), { headers }); + } else if (url.pathname.startsWith("/endpoint/")) { + const hostname = url.pathname.split("/")[2]; + const pubKey = await Utils.exportPublicKey(this._client.keyPair.publicKey); + + if (hostname === `${base32Encode(new TextEncoder().encode(pubKey.x)).toLowerCase().replace(/=+$/, "")}.${base32Encode(new TextEncoder().encode(pubKey.y)).toLowerCase().replace(/=+$/, "")}`) { + const body = this._client.config.reverseProxy + ? await (await fetch(this._client.config.reverseProxy)).text() + : (typeof this.handleCustomRequest === "undefined" ? "Hello World!" : await this.handleCustomRequest(new Request(`hydra://${hostname}/`))); + const signature = await Utils.signMessage(this._client.keyPair.privateKey, body); + + headers.set("hydra-signature", signature); + return new Response(body, { headers }); + } else { + if (this.hashLocks.has(hostname)) { + if (this._client.config.logLevel === "verbose") console.log(` ${hostname} Waiting for existing request with same hostname`); + await this.hashLocks.get(hostname); + } + if (hostname in this.cachedHostnames) return new Response(this.cachedHostnames[hostname].body, { headers: this.cachedHostnames[hostname].headers }); + + console.log(` ${hostname} Fetching endpoint response from peers`); + const responses = await this._client.rpcClient.fetch(`http://localhost/endpoint/${hostname}`); + + const processingPromise = new Promise((resolve, reject) => { + (async () => { + await Promise.all(responses.map(async (res) => { + try { + const response = await res; + if (response) { + const body = await response.text(); + const signature = response.headers.get("hydra-signature"); + if (signature !== null) { + const [xBase32, yBase32] = hostname.split("."); + if (await Utils.verifySignature(body, signature as Base64, { xBase32, yBase32 })) resolve(new Response(body, { headers: response.headers })); + } + } + } catch (e) { + const err = e as Error; + if (err.message !== "Hostname not found" && err.message !== "Promise timed out") console.error(e); + } + })); + reject(new Error("Hostname not found")); + })(); + }); + + this.hashLocks.set(hostname, processingPromise); + + let response: Response | undefined; + try { + response = await processingPromise; + } catch (e) { + const err = e as Error; + if (err.message === "Hstname not found") return new Response("Hostname not found", { headers, status: 404 }); + else throw err; + } finally { + this.hashLocks.delete(hostname); + } + const res = { body: await response.text(), headers: response.headers }; + this.cachedHostnames[hostname] = res; + return new Response(res.body, { headers: res.headers }); + } + } else if (url.pathname === "/block_height") { + headers.set("Content-Type", "application/json"); + headers.set("Cache-Control", "public, max-age=30"); + // return new Response(String(this._client.blockchain.lastBlock().height)); + } else { + return new Response("404 Page Not Found\n", { status: 404 }); + } + } catch (e) { + console.error("Internal Server Error", e); + return new Response("Internal Server Error", { status: 500 }); + } + return new Response("Something went wrong", { status: 500 }); + }; +} + +export default RPCServer; diff --git a/src/server.ts b/src/server.ts deleted file mode 100644 index e937966..0000000 --- a/src/server.ts +++ /dev/null @@ -1,389 +0,0 @@ -import Base32 from "npm:base32"; -import type Hydrafiles from "./hydrafiles.ts"; -// import { BLOCKSDIR } from "./block.ts"; -import File, { fileAttributesDefaults } from "./file.ts"; -import Utils from "./utils.ts"; -import { join } from "https://deno.land/std@0.224.0/path/mod.ts"; -import type Base64 from "npm:base64"; -import { HTTPPeer } from "./peers/HTTPPeers.ts"; -import { Message } from "./peers/RTCPeers.ts"; - -export const hashLocks = new Map>(); -const cachedHostnames: { [key: string]: { body: string; headers: Headers } } = {}; -const sockets: { id: number; socket: WebSocket }[] = []; - -export const handleRequest = async (req: Request, client: Hydrafiles): Promise => { - console.log(`Received Request: ${req.url}`); - const url = new URL(req.url); - const headers = new Headers(); - headers.set("Access-Control-Allow-Origin", "*"); - - try { - if (req.headers.get("upgrade") === "websocket") { - const { socket, response } = Deno.upgradeWebSocket(req); - sockets.push({ socket, id: 0 }); - - socket.addEventListener("message", ({ data }) => { - const message = JSON.parse(data) as Message | null; - if (message === null) return; - for (let i = 0; i < sockets.length; i++) { - if (sockets[i].socket !== socket && (!("to" in message) || message.to === sockets[i].id)) { - if (sockets[i].socket.readyState === 1) sockets[i].socket.send(data); - } else if ("from" in message) { - sockets[i].id = message.from; - } - } - }); - - return response; - } else if (url.pathname === "/" || url.pathname === undefined) { - headers.set("Content-Type", "text/html"); - headers.set("Cache-Control", "public, max-age=604800"); - return new Response(await client.fs.readFile("public/index.html") || "", { headers }); - } else if (url.pathname === "/favicon.ico") { - headers.set("Content-Type", "image/x-icon"); - headers.set("Cache-Control", "public, max-age=604800"); - return new Response(await client.fs.readFile("public/favicon.ico") || "", { headers }); - } else if (url.pathname === "/status") { - headers.set("Content-Type", "application/json"); - return new Response(JSON.stringify({ status: true }), { headers }); - } else if (url.pathname === "/hydrafiles-web.esm.js") { - headers.set("Content-Type", "application/javascript"); - headers.set("Cache-Control", "public, max-age=300"); - return new Response(await client.fs.readFile("build/hydrafiles-web.esm.js") || "", { headers }); - } else if (url.pathname === "/hydrafiles-web.esm.js.map") { - headers.set("Content-Type", "application/json"); - headers.set("Cache-Control", "public, max-age=300"); - return new Response(await client.fs.readFile("build/hydrafiles-web.esm.js.map") || "", { headers }); - } else if (url.pathname === "/demo.html") { - headers.set("Content-Type", "text/html"); - headers.set("Cache-Control", "public, max-age=300"); - return new Response(await client.fs.readFile("public/demo.html") || "", { headers }); - } else if (url.pathname === "/dashboard.html") { - headers.set("Content-Type", "text/html"); - headers.set("Cache-Control", "public, max-age=300"); - return new Response(await client.fs.readFile("public/dashboard.html") || "", { headers }); - } else if (url.pathname === "/peers") { - headers.set("Content-Type", "application/json"); - headers.set("Cache-Control", "public, max-age=300"); - return new Response(JSON.stringify(await client.http.getPeers()), { headers }); - } else if (url.pathname === "/info") { - headers.set("Content-Type", "application/json"); - headers.set("Cache-Control", "public, max-age=300"); - return new Response(JSON.stringify({ version: JSON.parse(await Deno.readTextFile("deno.jsonc")).version }), { headers }); - } else if (url.pathname.startsWith("/announce")) { - const host = url.searchParams.get("host"); - if (host === null) return new Response("No hosted given\n", { status: 401 }); - const knownNodes = await client.http.getPeers(); - if (knownNodes.find((node) => node.host === host) !== undefined) return new Response("Already known\n"); - await client.http.add(host); - return new Response("Announced\n"); - } else if (url.pathname?.startsWith("/download/")) { - const hash = url.pathname.split("/")[2]; - const fileId = url.pathname.split("/")[3] ?? ""; - const infohash = Array.from(decodeURIComponent(url.searchParams.get("info_hash") ?? "")).map((char) => char.charCodeAt(0).toString(16).padStart(2, "0")).join(""); - - if (hashLocks.has(hash)) { - if (client.config.logLevel === "verbose") console.log(` ${hash} Waiting for existing request with same hash`); - await hashLocks.get(hash); - } - const processingPromise = (async () => { - const file = await File.init({ hash, infohash }, client, true); - if (!file) throw new Error("Failed to build file"); - - if (fileId.length !== 0) { - const id = file.id; - if (id === undefined || id === null || id.length === 0) { - file.id = fileId; - file.save(); - } - } - - await file.getMetadata(); - let fileContent: { file: Uint8Array; signal: number } | false; - try { - fileContent = await file.getFile({ logDownloads: true }); - } catch (e) { - const err = e as { message: string }; - if (err.message === "Promise timed out") { - fileContent = false; - } else { - throw e; - } - } - - if (fileContent === false) { - file.found = false; - file.save(); - return new Response("404 File Not Found\n", { - status: 404, - }); - } - - headers.set("Content-Type", "application/octet-stream"); - headers.set("Cache-Control", "public, max-age=31536000"); - headers.set("Content-Length", fileContent.file.byteLength.toString()); - headers.set("Signal-Strength", String(fileContent.signal)); - console.log(` ${hash} Signal Strength:`, fileContent.signal, Utils.estimateHops(fileContent.signal)); - - headers.set("Content-Length", String(file.size)); - if (file.name !== undefined && file.name !== null) { - headers.set("Content-Disposition", `attachment; filename="${encodeURIComponent(file.name.replace(/[^a-zA-Z0-9._-]/g, "").replace(/\s+/g, " ").trim()).replace(/%20/g, " ").replace(/(\.\w+)$/, " [HYDRAFILES]$1")}"`); - } - - return new Response(fileContent.file, { headers }); - })(); - - hashLocks.set(hash, processingPromise); - - let response: Response; - try { - response = await processingPromise; - } finally { - hashLocks.delete(hash); - } - return response; - } else if (url.pathname?.startsWith("/infohash/")) { - const infohash = url.pathname.split("/")[2]; - - if (hashLocks.has(infohash)) { - console.log(` ${infohash} Waiting for existing request with same infohash`); - await hashLocks.get(infohash); - } - const processingPromise = (async () => { - const file = await File.init({ infohash }, client, true); - if (!file) throw new Error("Failed to build file"); - - await file.getMetadata(); - let fileContent: { file: Uint8Array; signal: number } | false; - try { - fileContent = await file.getFile({ logDownloads: true }); - } catch (e) { - const err = e as { message: string }; - if (err.message === "Promise timed out") { - fileContent = false; - } else { - throw e; - } - } - - if (fileContent === false) { - file.found = false; - file.save(); - return new Response("404 File Not Found\n", { - status: 404, - }); - } - - headers.set("Content-Type", "application/octet-stream"); - headers.set("Cache-Control", "public, max-age=31536000"); - - headers.set("Signal-Strength", String(fileContent.signal)); - console.log(` ${file.hash} Signal Strength:`, fileContent.signal, Utils.estimateHops(fileContent.signal)); - - headers.set("Content-Length", String(file.size)); - if (file.name !== null) headers.set("Content-Disposition", `attachment; filename="${encodeURIComponent(file.name).replace(/%20/g, " ").replace(/(\.\w+)$/, " [HYDRAFILES]$1")}"`); - - return new Response(fileContent.file, { headers }); - })(); - - hashLocks.set(infohash, processingPromise); - - try { - await processingPromise; - } finally { - hashLocks.delete(infohash); - } - } else if (url.pathname === "/upload") { - const uploadSecret = req.headers.get("x-hydra-upload-secret"); - if (uploadSecret !== client.config.uploadSecret) { - return new Response("401 Unauthorized\n", { status: 401 }); - } - - const form = await req.formData(); - const formData = { - hash: form.get("hash")?.toString(), - file: form.get("file") as globalThis.File | null, - }; - - if (typeof formData.hash === "undefined" || typeof formData.file === "undefined" || formData.file === null) return new Response("400 Bad Request\n", { status: 400 }); - - const hash = formData.hash[0]; - - const file = await File.init({ hash }, client, true); - if (!file) throw new Error("Failed to build file"); - if ((file.name === null || file.name.length === 0) && formData.file.name !== null) { - file.name = formData.file.name; - file.cacheFile(new Uint8Array(await formData.file.arrayBuffer())); - file.save(); - } - - console.log("Uploading", file.hash); - - if (await client.fs.exists(join("files", file.hash))) return new Response("200 OK\n"); - - if (!client.config.permaFiles.includes(hash)) client.config.permaFiles.push(hash); - await client.fs.writeFile("config.json", new TextEncoder().encode(JSON.stringify(client.config, null, 2))); - return new Response("200 OK\n"); - } else if (url.pathname === "/files") { - const rows = (await client.fileDB.select()).map((row) => { - const { downloadCount, found, ...rest } = row; - const _ = { downloadCount, found }; - return rest; - }); - headers.set("Content-Type", "application/json"); - headers.set("Cache-Control", "public, max-age=10800"); - return new Response(JSON.stringify(rows), { headers }); - } else if (url.pathname.startsWith("/file/")) { - const id = url.pathname.split("/")[2]; - let file: File | false; - try { - file = await File.init({ id }, client, true); - } catch (e) { - const err = e as Error; - if (err.message === "File not found") return new Response("File not found", { headers, status: 404 }); - else throw err; - } - - headers.set("Content-Type", "application/json"); - headers.set("Cache-Control", "public, max-age=10800"); - if (!file) return new Response("File not found", { headers, status: 404 }); - return new Response(JSON.stringify(fileAttributesDefaults(file)), { headers }); - } else if (url.pathname.startsWith("/endpoint/")) { - const hostname = url.pathname.split("/")[2]; - const pubKey = await Utils.exportPublicKey(client.keyPair.publicKey); - - if (hostname === `${Base32.encode(pubKey.x).toLowerCase().replaceAll("=", "")}.${Base32.encode(pubKey.y).toLowerCase().replaceAll("=", "")}`) { - const body = client.config.reverseProxy ? await (await fetch(client.config.reverseProxy)).text() : (typeof client.handleRequest === "undefined" ? "Hello World!" : await client.handleRequest(new Request(`hydra://${hostname}/`))); - const signature = await Utils.signMessage(client.keyPair.privateKey, body); - - headers.set("hydra-signature", signature); - return new Response(body, { headers }); - } else { - if (hashLocks.has(hostname)) { - if (client.config.logLevel === "verbose") console.log(` ${hostname} Waiting for existing request with same hostname`); - await hashLocks.get(hostname); - } - if (hostname in cachedHostnames) return new Response(cachedHostnames[hostname].body, { headers: cachedHostnames[hostname].headers }); - - console.log(` ${hostname} Fetching endpoint response from peers`); - const responses = await client.peers.fetch(`http://localhost/endpoint/${hostname}`); - - const processingPromise = new Promise((resolve, reject) => { - (async () => { - await Promise.all(responses.map(async (res) => { - try { - const response = await res; - if (response) { - const body = await response.text(); - const signature = response.headers.get("hydra-signature"); - if (signature !== null) { - const [xBase32, yBase32] = hostname.split("."); - if (await Utils.verifySignature(body, signature as Base64, { x: Base32.decode(xBase32), y: Base32.decode(yBase32) })) resolve(new Response(body, { headers: response.headers })); - } - } - } catch (e) { - const err = e as Error; - if (err.message !== "Hostname not found" && err.message !== "Promise timed out") console.error(e); - } - })); - reject(new Error("Hostname not found")); - })(); - }); - - hashLocks.set(hostname, processingPromise); - - let response: Response | undefined; - try { - response = await processingPromise; - } catch (e) { - const err = e as Error; - if (err.message === "Hstname not found") return new Response("Hostname not found", { headers, status: 404 }); - else throw err; - } finally { - hashLocks.delete(hostname); - } - const res = { body: await response.text(), headers: response.headers }; - cachedHostnames[hostname] = res; - return new Response(res.body, { headers: res.headers }); - } - - // } else if (url.pathname.startsWith("/block/")) { - // const blockHeight = url.pathname.split("/")[2]; - // headers.set("Content-Type", "application/json"); - // // "Cache-Control": "public, max-age=" + (Number(blockHeight) > client.blockchain.lastBlock().height ? 0 : 604800), - // const block = await client.fs.readFile(join(BLOCKSDIR, blockHeight)); - // return new Response(block, { headers }); - } else if (url.pathname === "/block_height") { - headers.set("Content-Type", "application/json"); - headers.set("Cache-Control", "public, max-age=30"); - // return new Response(String(client.blockchain.lastBlock().height)); - } else { - return new Response("404 Page Not Found\n", { status: 404 }); - } - } catch (e) { - console.error("Internal Server Error", e); - return new Response("Internal Server Error", { status: 500 }); - } - return new Response("Something went wrong", { status: 500 }); -}; - -const onListen = (client: Hydrafiles): void => { - console.log(`Server running at ${client.config.publicHostname}/`); - - const handleListen = async (): Promise => { - console.log("Testing network connection"); - const file = await client.peers.downloadFile("04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f"); - if (file === false) console.error("Download test failed, cannot connect to network"); - else { - console.log("Connected to network"); - - if (Utils.isIp(client.config.publicHostname) && Utils.isPrivateIP(client.config.publicHostname)) console.error("Public hostname is a private IP address, cannot announce to other nodes"); - else { - console.log(`Testing downloads ${client.config.publicHostname}/download/04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f`); - - console.log("Testing file build"); - const file = await File.init({ hash: "04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f" }, client); - if (!file) console.error("Failed to build file"); - else { - console.log("Testing connectivity"); - const response = await client.http.downloadFromPeer(await HTTPPeer.init({ host: client.config.publicHostname }, client.http._db), file); - if (response === false) console.error(" 04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f ERROR: Failed to download file from self"); - else { - console.log(" 04aa07009174edc6f03224f003a435bcdc9033d2c52348f3a35fbb342ea82f6f Test Succeeded"); - console.log("Announcing HTTP server to nodes"); - client.peers.announceHTTP(); - } - await client.http.add(client.config.publicHostname); - } - } - } - }; - handleListen().catch(console.error); -}; - -const startServer = (client: Hydrafiles): void => { - if (typeof window !== "undefined") return; - console.log("Starting server"); - - let port = client.config.port; - while (true) { - try { - Deno.serve({ - port, - hostname: client.config.hostname, - onListen({ hostname, port }): void { - onListen(client); - console.log(`Server started at ${hostname}:${port}`); - }, - handler: async (req: Request): Promise => await handleRequest(req, client), - }); - return; - } catch (e) { - const err = e as Error; - if (err.name !== "AddrInUse") throw err; - port++; - } - } -}; -export default startServer; diff --git a/src/start.ts b/src/start.ts index f18fec8..f99d26f 100644 --- a/src/start.ts +++ b/src/start.ts @@ -1,4 +1,4 @@ -import { existsSync } from "jsr:@std/fs@0.221/exists"; +import { existsSync } from "jsr:@std/fs/exists"; import Hydrafiles from "./hydrafiles.ts"; const configPath = Deno.args[0] ?? "config.json"; diff --git a/src/utils.ts b/src/utils.ts index 9b63e1c..1362531 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,6 +2,7 @@ import { crypto } from "jsr:@std/crypto"; import { encodeHex } from "jsr:@std/encoding/hex"; import type Hydrafiles from "./hydrafiles.ts"; import { join } from "https://deno.land/std@0.224.0/path/mod.ts"; +import { decodeBase32 } from "jsr:@std/encoding@^1.0.5/base32"; export type Base64 = string & { __brand: "Base64" }; export type NonNegativeNumber = number & { readonly brand: unique symbol }; @@ -257,13 +258,20 @@ class Utils { ); return this.bufferToBase64(signature); } - static async verifySignature(message: string, signature: Base64, pubKey: { x: string; y: string }): Promise { + static async verifySignature(message: string, signature: Base64, pubKey: { x: string; y: string } | { xBase32: string; yBase32: string }): Promise { const encoder = new TextEncoder(); const data = encoder.encode(message); + const decodedPubKey = "xBase32" in pubKey + ? { + x: new TextDecoder().decode(decodeBase32(pubKey.xBase32)), + y: new TextDecoder().decode(decodeBase32(pubKey.yBase32)), + } + : pubKey; + const importedPublicKey = await crypto.subtle.importKey( "jwk", - this.buildJWT(pubKey), + this.buildJWT(decodedPubKey), { name: "ECDSA", namedCurve: "P-256" }, true, ["verify"],