From 3d7fb37f313105d548faccc6389156f8ee0fd89a Mon Sep 17 00:00:00 2001 From: DimaDemchenko Date: Mon, 26 Feb 2024 17:27:16 +0200 Subject: [PATCH 1/6] segment validation --- .../p2p-media-loader-core/src/p2p/peer-protocol.ts | 5 ++++- packages/p2p-media-loader-core/src/p2p/peer.ts | 13 ++++++++++--- packages/p2p-media-loader-core/src/types.d.ts | 1 + 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts b/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts index 3527ead8..ad6f8525 100644 --- a/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts +++ b/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts @@ -5,7 +5,10 @@ import { Settings } from "../types"; export type PeerSettings = Pick< Settings, - "p2pNotReceivingBytesTimeoutMs" | "webRtcMaxMessageSize" | "p2pErrorRetries" + | "p2pNotReceivingBytesTimeoutMs" + | "webRtcMaxMessageSize" + | "p2pErrorRetries" + | "validateP2Psegment" >; export class PeerProtocol { diff --git a/packages/p2p-media-loader-core/src/p2p/peer.ts b/packages/p2p-media-loader-core/src/p2p/peer.ts index 698afc96..a42cc96a 100644 --- a/packages/p2p-media-loader-core/src/p2p/peer.ts +++ b/packages/p2p-media-loader-core/src/p2p/peer.ts @@ -45,6 +45,7 @@ export class Peer { this.id = Peer.getPeerIdFromConnection(connection); this.peerProtocol = new PeerProtocol(connection, settings, { onSegmentChunkReceived: this.onSegmentChunkReceived, + // eslint-disable-next-line @typescript-eslint/no-misused-promises onCommandReceived: this.onCommandReceived, }); @@ -62,7 +63,7 @@ export class Peer { if (this.httpLoadingSegments.has(externalId)) return "http-loading"; } - private onCommandReceived = (command: Command.PeerCommand) => { + private onCommandReceived = async (command: Command.PeerCommand) => { switch (command.c) { case PeerCommandType.SegmentsAnnouncement: this.loadedSegments = new Set(command.l); @@ -106,9 +107,15 @@ export class Peer { return; } - const isWrongBytes = request.loadedBytes !== request.totalBytes; + let isValid = true; + if (this.settings.validateP2Psegment) { + isValid = await this.settings.validateP2Psegment( + request.segment.url, + request.segment.byteRange, + ); + } - if (isWrongBytes) { + if (!isValid) { request.clearLoadedBytes(); this.cancelSegmentDownloading("peer-response-bytes-mismatch"); this.destroy(); diff --git a/packages/p2p-media-loader-core/src/types.d.ts b/packages/p2p-media-loader-core/src/types.d.ts index 67b3ec8e..01230173 100644 --- a/packages/p2p-media-loader-core/src/types.d.ts +++ b/packages/p2p-media-loader-core/src/types.d.ts @@ -53,6 +53,7 @@ export type Settings = { httpNotReceivingBytesTimeoutMs: number; httpErrorRetries: number; p2pErrorRetries: number; + validateP2Psegment?: (url: string, byteRange?: ByteRange) => Promise; }; export type CoreEventHandlers = { From 22bdd179d9a12a6bd1f0f230479856809ccfea7a Mon Sep 17 00:00:00 2001 From: DimaDemchenko Date: Mon, 26 Feb 2024 21:45:41 +0200 Subject: [PATCH 2/6] improvements --- .../src/p2p/peer-protocol.ts | 6 ++-- .../p2p-media-loader-core/src/p2p/peer.ts | 33 ++++++++++++------- .../src/requests/request.ts | 11 ++++--- packages/p2p-media-loader-core/src/types.d.ts | 4 +-- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts b/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts index ad6f8525..9b245ee7 100644 --- a/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts +++ b/packages/p2p-media-loader-core/src/p2p/peer-protocol.ts @@ -1,14 +1,14 @@ import { PeerConnection } from "bittorrent-tracker"; -import * as Command from "./commands"; -import * as Utils from "../utils/utils"; import { Settings } from "../types"; +import * as Utils from "../utils/utils"; +import * as Command from "./commands"; export type PeerSettings = Pick< Settings, | "p2pNotReceivingBytesTimeoutMs" | "webRtcMaxMessageSize" | "p2pErrorRetries" - | "validateP2Psegment" + | "validateP2PSegment" >; export class PeerProtocol { diff --git a/packages/p2p-media-loader-core/src/p2p/peer.ts b/packages/p2p-media-loader-core/src/p2p/peer.ts index a42cc96a..42fbd664 100644 --- a/packages/p2p-media-loader-core/src/p2p/peer.ts +++ b/packages/p2p-media-loader-core/src/p2p/peer.ts @@ -1,16 +1,16 @@ import { PeerConnection } from "bittorrent-tracker"; -import { PeerProtocol, PeerSettings } from "./peer-protocol"; +import debug from "debug"; import { + PeerRequestErrorType, Request, RequestControls, RequestError, - PeerRequestErrorType, RequestInnerErrorType, } from "../requests/request"; -import * as Command from "./commands"; import { Segment } from "../types"; import * as Utils from "../utils/utils"; -import debug from "debug"; +import * as Command from "./commands"; +import { PeerProtocol, PeerSettings } from "./peer-protocol"; const { PeerCommandType } = Command; type PeerEventHandlers = { @@ -86,7 +86,9 @@ export class Peer { request.setTotalBytes(command.s); } else if (request.totalBytes - request.loadedBytes !== command.s) { request.clearLoadedBytes(); - this.cancelSegmentDownloading("peer-response-bytes-mismatch"); + this.cancelSegmentDownloading( + "peer-response-bytes-length-mismatch", + ); this.destroy(); } } @@ -107,17 +109,24 @@ export class Peer { return; } - let isValid = true; - if (this.settings.validateP2Psegment) { - isValid = await this.settings.validateP2Psegment( + const isWrongBytes = request.loadedBytes !== request.totalBytes; + + if (isWrongBytes) { + request.clearLoadedBytes(); + this.cancelSegmentDownloading("peer-response-bytes-length-mismatch"); + this.destroy(); + return; + } + + const isValid = + (await this.settings.validateP2PSegment?.( request.segment.url, request.segment.byteRange, - ); - } + )) ?? true; if (!isValid) { request.clearLoadedBytes(); - this.cancelSegmentDownloading("peer-response-bytes-mismatch"); + this.cancelSegmentDownloading("p2p-segment-validation-failed"); this.destroy(); return; } @@ -152,7 +161,7 @@ export class Peer { if (isOverflow) { request.clearLoadedBytes(); - this.cancelSegmentDownloading("peer-response-bytes-mismatch"); + this.cancelSegmentDownloading("peer-response-bytes-length-mismatch"); this.destroy(); return; } diff --git a/packages/p2p-media-loader-core/src/requests/request.ts b/packages/p2p-media-loader-core/src/requests/request.ts index 1931b297..9b88784b 100644 --- a/packages/p2p-media-loader-core/src/requests/request.ts +++ b/packages/p2p-media-loader-core/src/requests/request.ts @@ -1,8 +1,8 @@ -import { Segment, Playback, BandwidthCalculators } from "../types"; +import debug from "debug"; +import { BandwidthCalculators, Playback, Segment } from "../types"; +import * as LoggerUtils from "../utils/logger"; import * as StreamUtils from "../utils/stream"; import * as Utils from "../utils/utils"; -import * as LoggerUtils from "../utils/logger"; -import debug from "debug"; export type LoadProgress = { startTimestamp: number; @@ -323,10 +323,11 @@ export type HttpRequestErrorType = | "http-unexpected-status-code"; export type PeerRequestErrorType = - | "peer-response-bytes-mismatch" + | "peer-response-bytes-length-mismatch" | "peer-protocol-violation" | "peer-segment-absent" - | "peer-closed"; + | "peer-closed" + | "p2p-segment-validation-failed"; type RequestErrorType = | RequestInnerErrorType diff --git a/packages/p2p-media-loader-core/src/types.d.ts b/packages/p2p-media-loader-core/src/types.d.ts index 01230173..e7661f86 100644 --- a/packages/p2p-media-loader-core/src/types.d.ts +++ b/packages/p2p-media-loader-core/src/types.d.ts @@ -1,5 +1,5 @@ -import { RequestAttempt } from "./requests/request"; import { BandwidthCalculator } from "./bandwidth-calculator"; +import { RequestAttempt } from "./requests/request"; export type StreamType = "main" | "secondary"; @@ -53,7 +53,7 @@ export type Settings = { httpNotReceivingBytesTimeoutMs: number; httpErrorRetries: number; p2pErrorRetries: number; - validateP2Psegment?: (url: string, byteRange?: ByteRange) => Promise; + validateP2PSegment?: (url: string, byteRange?: ByteRange) => Promise; }; export type CoreEventHandlers = { From b967ea8fe5c741ad9a686e7f7e4af9115ea9ca58 Mon Sep 17 00:00:00 2001 From: DimaDemchenko Date: Tue, 27 Feb 2024 10:38:32 +0200 Subject: [PATCH 3/6] test func --- packages/p2p-media-loader-core/src/core.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/p2p-media-loader-core/src/core.ts b/packages/p2p-media-loader-core/src/core.ts index 65ee0da9..d56eda74 100644 --- a/packages/p2p-media-loader-core/src/core.ts +++ b/packages/p2p-media-loader-core/src/core.ts @@ -8,6 +8,7 @@ import { CoreEventHandlers, BandwidthCalculators, StreamDetails, + ByteRange, } from "./types"; import * as StreamUtils from "./utils/stream"; import { BandwidthCalculator } from "./bandwidth-calculator"; @@ -31,6 +32,17 @@ export class Core { httpNotReceivingBytesTimeoutMs: 1000, httpErrorRetries: 3, p2pErrorRetries: 3, + validateP2PSegment: async ( + url: string, + byteRange?: ByteRange | undefined, + ) => { + await new Promise((resolve) => setTimeout(resolve, 3000)); + + console.log("validateP2PSegment", url, byteRange); + + //return Math.random() > 0.5; + return true; + }, }; private readonly bandwidthCalculators: BandwidthCalculators = { all: new BandwidthCalculator(), From 8df26518e0e75ce211048278824599813592dbe1 Mon Sep 17 00:00:00 2001 From: DimaDemchenko Date: Wed, 28 Feb 2024 11:32:38 +0200 Subject: [PATCH 4/6] error type --- packages/p2p-media-loader-core/src/p2p/peer.ts | 4 +++- packages/p2p-media-loader-core/src/requests/request.ts | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/p2p-media-loader-core/src/p2p/peer.ts b/packages/p2p-media-loader-core/src/p2p/peer.ts index 42fbd664..34846179 100644 --- a/packages/p2p-media-loader-core/src/p2p/peer.ts +++ b/packages/p2p-media-loader-core/src/p2p/peer.ts @@ -184,7 +184,9 @@ export class Peer { abort: (error) => { if (!this.downloadingContext) return; const { request } = this.downloadingContext; - this.sendCancelSegmentRequestCommand(request.segment); + if (error.type !== "p2p-segment-validation-failed") { + this.sendCancelSegmentRequestCommand(request.segment); + } this.downloadingContext = undefined; this.downloadingErrors.push(error); diff --git a/packages/p2p-media-loader-core/src/requests/request.ts b/packages/p2p-media-loader-core/src/requests/request.ts index 9b88784b..9486b7a0 100644 --- a/packages/p2p-media-loader-core/src/requests/request.ts +++ b/packages/p2p-media-loader-core/src/requests/request.ts @@ -138,7 +138,9 @@ export class Request { requestData: StartRequestParameters, controls: { notReceivingBytesTimeoutMs?: number; - abort: (errorType: RequestError) => void; + abort: ( + errorType: RequestError, + ) => void; }, ): RequestControls { if (this._status === "succeed") { From e33a63b7ac20911f5d1cb6653c8d7a654d322c84 Mon Sep 17 00:00:00 2001 From: DimaDemchenko Date: Wed, 28 Feb 2024 16:00:33 +0200 Subject: [PATCH 5/6] fixed p2p segment validation --- packages/p2p-media-loader-core/src/core.ts | 12 ------------ packages/p2p-media-loader-core/src/p2p/peer.ts | 18 +++++++++++------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/packages/p2p-media-loader-core/src/core.ts b/packages/p2p-media-loader-core/src/core.ts index d56eda74..65ee0da9 100644 --- a/packages/p2p-media-loader-core/src/core.ts +++ b/packages/p2p-media-loader-core/src/core.ts @@ -8,7 +8,6 @@ import { CoreEventHandlers, BandwidthCalculators, StreamDetails, - ByteRange, } from "./types"; import * as StreamUtils from "./utils/stream"; import { BandwidthCalculator } from "./bandwidth-calculator"; @@ -32,17 +31,6 @@ export class Core { httpNotReceivingBytesTimeoutMs: 1000, httpErrorRetries: 3, p2pErrorRetries: 3, - validateP2PSegment: async ( - url: string, - byteRange?: ByteRange | undefined, - ) => { - await new Promise((resolve) => setTimeout(resolve, 3000)); - - console.log("validateP2PSegment", url, byteRange); - - //return Math.random() > 0.5; - return true; - }, }; private readonly bandwidthCalculators: BandwidthCalculators = { all: new BandwidthCalculator(), diff --git a/packages/p2p-media-loader-core/src/p2p/peer.ts b/packages/p2p-media-loader-core/src/p2p/peer.ts index 34846179..ccc268c4 100644 --- a/packages/p2p-media-loader-core/src/p2p/peer.ts +++ b/packages/p2p-media-loader-core/src/p2p/peer.ts @@ -86,6 +86,7 @@ export class Peer { request.setTotalBytes(command.s); } else if (request.totalBytes - request.loadedBytes !== command.s) { request.clearLoadedBytes(); + this.sendCancelSegmentRequestCommand(request.segment); this.cancelSegmentDownloading( "peer-response-bytes-length-mismatch", ); @@ -95,12 +96,14 @@ export class Peer { break; case PeerCommandType.SegmentDataSendingCompleted: { - if (!this.downloadingContext?.isSegmentDataCommandReceived) return; + const downloadingContext = this.downloadingContext; - const { request, controls } = this.downloadingContext; + if (!downloadingContext?.isSegmentDataCommandReceived) return; + + const { request, controls } = downloadingContext; const isWrongSegment = - this.downloadingContext.request.segment.externalId !== command.i; + downloadingContext.request.segment.externalId !== command.i; if (isWrongSegment) { request.clearLoadedBytes(); @@ -124,6 +127,8 @@ export class Peer { request.segment.byteRange, )) ?? true; + if (this.downloadingContext !== downloadingContext) return; + if (!isValid) { request.clearLoadedBytes(); this.cancelSegmentDownloading("p2p-segment-validation-failed"); @@ -184,11 +189,10 @@ export class Peer { abort: (error) => { if (!this.downloadingContext) return; const { request } = this.downloadingContext; - if (error.type !== "p2p-segment-validation-failed") { - this.sendCancelSegmentRequestCommand(request.segment); - } - this.downloadingContext = undefined; + + this.sendCancelSegmentRequestCommand(request.segment); this.downloadingErrors.push(error); + this.downloadingContext = undefined; const timeoutErrors = this.downloadingErrors.filter( (error) => error.type === "bytes-receiving-timeout", From b720bf240ac7a59e7887a97f771f7dfc5b7cc5c4 Mon Sep 17 00:00:00 2001 From: Andriy Lysnevych Date: Thu, 29 Feb 2024 08:51:44 +0200 Subject: [PATCH 6/6] Revert abort callback type --- packages/p2p-media-loader-core/src/requests/request.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/p2p-media-loader-core/src/requests/request.ts b/packages/p2p-media-loader-core/src/requests/request.ts index 9486b7a0..9b88784b 100644 --- a/packages/p2p-media-loader-core/src/requests/request.ts +++ b/packages/p2p-media-loader-core/src/requests/request.ts @@ -138,9 +138,7 @@ export class Request { requestData: StartRequestParameters, controls: { notReceivingBytesTimeoutMs?: number; - abort: ( - errorType: RequestError, - ) => void; + abort: (errorType: RequestError) => void; }, ): RequestControls { if (this._status === "succeed") {