From 00fa514b030b311a9e493391cbb938b2a04667ed Mon Sep 17 00:00:00 2001 From: LuanRT Date: Fri, 6 Jan 2023 03:06:49 -0300 Subject: [PATCH] feat: add support for generating sessions locally (#277) * feat: add visitor data proto * feat: add support for generating session data locally * chore: add test --- src/core/Session.ts | 162 +++++++++++++++++++++++++----- src/proto/index.ts | 7 +- src/proto/youtube.proto | 17 ++-- src/proto/youtube.ts | 211 ++++++++++++++++++++++++++-------------- src/utils/Constants.ts | 4 +- test/main.test.ts | 5 + 6 files changed, 302 insertions(+), 104 deletions(-) diff --git a/src/core/Session.ts b/src/core/Session.ts index 4741ed329..21d50b4f7 100644 --- a/src/core/Session.ts +++ b/src/core/Session.ts @@ -1,12 +1,13 @@ import UniversalCache from '../utils/Cache'; -import Constants from '../utils/Constants'; +import Constants, { CLIENTS } from '../utils/Constants'; import EventEmitterLike from '../utils/EventEmitterLike'; import Actions from './Actions'; import Player from './Player'; import HTTPClient, { FetchFunction } from '../utils/HTTPClient'; -import { DeviceCategory, getRandomUserAgent, InnertubeError, SessionError } from '../utils/Utils'; +import { DeviceCategory, generateRandomString, getRandomUserAgent, InnertubeError, SessionError } from '../utils/Utils'; import OAuth, { Credentials, OAuthAuthErrorEventHandler, OAuthAuthEventHandler, OAuthAuthPendingEventHandler } from './OAuth'; +import Proto from '../proto'; export enum ClientType { WEB = 'WEB', @@ -21,7 +22,7 @@ export interface Context { client: { hl: string; gl: string; - remoteHost: string; + remoteHost?: string; screenDensityFloat: number; screenHeightPoints: number; screenPixelDensity: number; @@ -38,8 +39,8 @@ export interface Context { clientFormFactor: string; userInterfaceTheme: string; timeZone: string; - browserName: string; - browserVersion: string; + browserName?: string; + browserVersion?: string; originalUrl: string; deviceMake: string; deviceModel: string; @@ -58,25 +59,72 @@ export interface Context { } export interface SessionOptions { + /** + * Language. + */ lang?: string; + /** + * Geolocation. + */ location?: string; + /** + * The account index to use. This is useful if you have multiple accounts logged in. + * **NOTE:** + * Only works if you are signed in with cookies. + */ account_index?: number; + /** + * Specifies whether to retrieve the JS player. Disabling this will make session creation faster. + * **NOTE:** Deciphering formats is not possible without the JS player. + */ retrieve_player?: boolean; + /** + * Specifies whether to enable safety mode. This will prevent the session from loading any potentially unsafe content. + */ enable_safety_mode?: boolean; + /** + * Specifies whether to generate the session data locally or retrieve it from YouTube. + * This can be useful if you need more performance. + */ + generate_session_locally?: boolean; + /** + * Platform to use for the session. + */ device_category?: DeviceCategory; + /** + * InnerTube client type. + */ client_type?: ClientType; + /** + * The time zone. + */ timezone?: string; + /** + * Used to cache the deciphering functions from the JS player. + */ cache?: UniversalCache; + /** + * YouTube cookies. + */ cookie?: string; + /** + * Fetch function to use. + */ fetch?: FetchFunction; } +export interface SessionData { + context: Context; + api_key: string; + api_version: string; +} + export default class Session extends EventEmitterLike { - #api_version; - #key; - #context; - #account_index; - #player; + #api_version: string; + #key: string; + #context: Context; + #account_index: number; + #player?: Player; oauth: OAuth; http: HTTPClient; @@ -121,6 +169,7 @@ export default class Session extends EventEmitterLike { options.location, options.account_index, options.enable_safety_mode, + options.generate_session_locally, options.device_category, options.client_type, options.timezone, @@ -135,30 +184,49 @@ export default class Session extends EventEmitterLike { } static async getSessionData( - lang = 'en-US', + lang = '', location = '', account_index = 0, enable_safety_mode = false, + generate_session_locally = false, device_category: DeviceCategory = 'desktop', client_name: ClientType = ClientType.WEB, tz: string = Intl.DateTimeFormat().resolvedOptions().timeZone, fetch: FetchFunction = globalThis.fetch ) { + let session_data: SessionData; + + if (generate_session_locally) { + session_data = this.#generateSessionData({ lang, location, time_zone: tz, device_category, client_name, enable_safety_mode }); + } else { + session_data = await this.#retrieveSessionData({ lang, location, time_zone: tz, device_category, client_name, enable_safety_mode }, fetch); + } + + return { ...session_data, account_index }; + } + + static async #retrieveSessionData(options: { + lang: string; + location: string; + time_zone: string; + device_category: string; + client_name: string; + enable_safety_mode: boolean; + }, fetch: FetchFunction = globalThis.fetch): Promise { const url = new URL('/sw.js_data', Constants.URLS.YT_BASE); const res = await fetch(url, { headers: { - 'accept-language': lang, + 'accept-language': options.lang || 'en-US', 'user-agent': getRandomUserAgent('desktop'), 'accept': '*/*', 'referer': 'https://www.youtube.com/sw.js', - 'cookie': `PREF=tz=${tz.replace('/', '.')}` + 'cookie': `PREF=tz=${options.time_zone.replace('/', '.')}` } }); - if (!res.ok) { - throw new SessionError(`Failed to get session data: ${res.status}`); - } + if (!res.ok) + throw new SessionError(`Failed to retrieve session data: ${res.status}`); const text = await res.text(); const data = JSON.parse(text.replace(/^\)\]\}'/, '')); @@ -172,22 +240,22 @@ export default class Session extends EventEmitterLike { const context: Context = { client: { hl: device_info[0], - gl: location || device_info[2], + gl: options.location || device_info[2], remoteHost: device_info[3], screenDensityFloat: 1, - screenHeightPoints: 720, + screenHeightPoints: 1080, screenPixelDensity: 1, - screenWidthPoints: 1280, + screenWidthPoints: 1920, visitorData: device_info[13], userAgent: device_info[14], - clientName: client_name, + clientName: options.client_name, clientVersion: device_info[16], osName: device_info[17], osVersion: device_info[18], - platform: device_category.toUpperCase(), + platform: options.device_category.toUpperCase(), clientFormFactor: 'UNKNOWN_FORM_FACTOR', userInterfaceTheme: 'USER_INTERFACE_THEME_LIGHT', - timeZone: device_info[79], + timeZone: device_info[79] || options.time_zone, browserName: device_info[86], browserVersion: device_info[87], originalUrl: Constants.URLS.YT_BASE, @@ -196,7 +264,53 @@ export default class Session extends EventEmitterLike { utcOffsetMinutes: new Date().getTimezoneOffset() }, user: { - enableSafetyMode: enable_safety_mode, + enableSafetyMode: options.enable_safety_mode, + lockedSafetyMode: false + }, + request: { + useSsl: true + } + }; + + return { context, api_key, api_version }; + } + + static #generateSessionData(options: { + lang: string; + location: string; + time_zone: string; + device_category: DeviceCategory; + client_name: string; + enable_safety_mode: boolean + }): SessionData { + const id = generateRandomString(11); + const timestamp = Math.floor(Date.now() / 1000); + + const context: Context = { + client: { + hl: options.lang || 'en', + gl: options.location || 'US', + screenDensityFloat: 1, + screenHeightPoints: 1080, + screenPixelDensity: 1, + screenWidthPoints: 1920, + visitorData: Proto.encodeVisitorData(id, timestamp), + userAgent: getRandomUserAgent('desktop'), + clientName: options.client_name, + clientVersion: CLIENTS.WEB.VERSION, + osName: 'Windows', + osVersion: '10.0', + platform: options.device_category.toUpperCase(), + clientFormFactor: 'UNKNOWN_FORM_FACTOR', + userInterfaceTheme: 'USER_INTERFACE_THEME_LIGHT', + timeZone: options.time_zone, + originalUrl: Constants.URLS.YT_BASE, + deviceMake: '', + deviceModel: '', + utcOffsetMinutes: new Date().getTimezoneOffset() + }, + user: { + enableSafetyMode: options.enable_safety_mode, lockedSafetyMode: false }, request: { @@ -204,7 +318,7 @@ export default class Session extends EventEmitterLike { } }; - return { context, api_key, api_version, account_index }; + return { context, api_key: CLIENTS.WEB.API_KEY, api_version: CLIENTS.WEB.API_VERSION }; } async signIn(credentials?: Credentials): Promise { diff --git a/src/proto/index.ts b/src/proto/index.ts index c9ba01219..796e8a288 100644 --- a/src/proto/index.ts +++ b/src/proto/index.ts @@ -2,9 +2,14 @@ import { CLIENTS } from '../utils/Constants'; import { u8ToBase64 } from '../utils/Utils'; import { VideoMetadata } from '../core/Studio'; -import { ChannelAnalytics, CreateCommentParams, GetCommentsSectionParams, InnertubePayload, LiveMessageParams, MusicSearchFilter, NotificationPreferences, PeformCommentActionParams, SearchFilter, SearchFilter_Filters } from './youtube'; +import { ChannelAnalytics, CreateCommentParams, GetCommentsSectionParams, InnertubePayload, LiveMessageParams, MusicSearchFilter, NotificationPreferences, PeformCommentActionParams, SearchFilter, SearchFilter_Filters, VisitorData } from './youtube'; class Proto { + static encodeVisitorData(id: string, timestamp: number): string { + const buf = VisitorData.toBinary({ id, timestamp }); + return encodeURIComponent(u8ToBase64(buf).replace(/\+/g, '-').replace(/\//g, '_')); + } + static encodeChannelAnalyticsParams(channel_id: string): string { const buf = ChannelAnalytics.toBinary({ params: { diff --git a/src/proto/youtube.proto b/src/proto/youtube.proto index 3ee3585c8..c14e168ff 100644 --- a/src/proto/youtube.proto +++ b/src/proto/youtube.proto @@ -3,12 +3,9 @@ syntax = "proto2"; package youtube; -message ChannelAnalytics { - message Params { - required string channel_id = 1001; - } - - required Params params = 32; +message VisitorData { + required string id = 1; + required int32 timestamp = 5; } message InnertubePayload { @@ -91,6 +88,14 @@ message InnertubePayload { optional VideoThumbnail video_thumbnail = 20; } +message ChannelAnalytics { + message Params { + required string channel_id = 1001; + } + + required Params params = 32; +} + message SoundInfoParams { message Sound { message Params { diff --git a/src/proto/youtube.ts b/src/proto/youtube.ts index ef8c3f6cb..86a5e84fb 100644 --- a/src/proto/youtube.ts +++ b/src/proto/youtube.ts @@ -15,22 +15,17 @@ import { reflectionMergePartial } from "@protobuf-ts/runtime"; import { MESSAGE_TYPE } from "@protobuf-ts/runtime"; import { MessageType } from "@protobuf-ts/runtime"; /** - * @generated from protobuf message youtube.ChannelAnalytics + * @generated from protobuf message youtube.VisitorData */ -export interface ChannelAnalytics { +export interface VisitorData { /** - * @generated from protobuf field: youtube.ChannelAnalytics.Params params = 32; + * @generated from protobuf field: string id = 1; */ - params?: ChannelAnalytics_Params; -} -/** - * @generated from protobuf message youtube.ChannelAnalytics.Params - */ -export interface ChannelAnalytics_Params { + id: string; /** - * @generated from protobuf field: string channel_id = 1001; + * @generated from protobuf field: int32 timestamp = 5; */ - channelId: string; + timestamp: number; } /** * @generated from protobuf message youtube.InnertubePayload @@ -213,6 +208,24 @@ export interface InnertubePayload_VideoThumbnail_Thumbnail { */ imageData: Uint8Array; } +/** + * @generated from protobuf message youtube.ChannelAnalytics + */ +export interface ChannelAnalytics { + /** + * @generated from protobuf field: youtube.ChannelAnalytics.Params params = 32; + */ + params?: ChannelAnalytics_Params; +} +/** + * @generated from protobuf message youtube.ChannelAnalytics.Params + */ +export interface ChannelAnalytics_Params { + /** + * @generated from protobuf field: string channel_id = 1001; + */ + channelId: string; +} /** * @generated from protobuf message youtube.SoundInfoParams */ @@ -646,73 +659,30 @@ export interface SearchFilter_Filters { featuresVr180?: number; } // @generated message type with reflection information, may provide speed optimized methods -class ChannelAnalytics$Type extends MessageType { +class VisitorData$Type extends MessageType { constructor() { - super("youtube.ChannelAnalytics", [ - { no: 32, name: "params", kind: "message", T: () => ChannelAnalytics_Params } + super("youtube.VisitorData", [ + { no: 1, name: "id", kind: "scalar", T: 9 /*ScalarType.STRING*/ }, + { no: 5, name: "timestamp", kind: "scalar", T: 5 /*ScalarType.INT32*/ } ]); } - create(value?: PartialMessage): ChannelAnalytics { - const message = {}; + create(value?: PartialMessage): VisitorData { + const message = { id: "", timestamp: 0 }; globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); if (value !== undefined) - reflectionMergePartial(this, message, value); + reflectionMergePartial(this, message, value); return message; } - internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics): ChannelAnalytics { + internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: VisitorData): VisitorData { let message = target ?? this.create(), end = reader.pos + length; while (reader.pos < end) { let [fieldNo, wireType] = reader.tag(); switch (fieldNo) { - case /* youtube.ChannelAnalytics.Params params */ 32: - message.params = ChannelAnalytics_Params.internalBinaryRead(reader, reader.uint32(), options, message.params); + case /* string id */ 1: + message.id = reader.string(); break; - default: - let u = options.readUnknownField; - if (u === "throw") - throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); - let d = reader.skip(wireType); - if (u !== false) - (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); - } - } - return message; - } - internalBinaryWrite(message: ChannelAnalytics, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { - /* youtube.ChannelAnalytics.Params params = 32; */ - if (message.params) - ChannelAnalytics_Params.internalBinaryWrite(message.params, writer.tag(32, WireType.LengthDelimited).fork(), options).join(); - let u = options.writeUnknownFields; - if (u !== false) - (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); - return writer; - } -} -/** - * @generated MessageType for protobuf message youtube.ChannelAnalytics - */ -export const ChannelAnalytics = new ChannelAnalytics$Type(); -// @generated message type with reflection information, may provide speed optimized methods -class ChannelAnalytics_Params$Type extends MessageType { - constructor() { - super("youtube.ChannelAnalytics.Params", [ - { no: 1001, name: "channel_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ } - ]); - } - create(value?: PartialMessage): ChannelAnalytics_Params { - const message = { channelId: "" }; - globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); - if (value !== undefined) - reflectionMergePartial(this, message, value); - return message; - } - internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics_Params): ChannelAnalytics_Params { - let message = target ?? this.create(), end = reader.pos + length; - while (reader.pos < end) { - let [fieldNo, wireType] = reader.tag(); - switch (fieldNo) { - case /* string channel_id */ 1001: - message.channelId = reader.string(); + case /* int32 timestamp */ 5: + message.timestamp = reader.int32(); break; default: let u = options.readUnknownField; @@ -725,10 +695,13 @@ class ChannelAnalytics_Params$Type extends MessageType } return message; } - internalBinaryWrite(message: ChannelAnalytics_Params, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { - /* string channel_id = 1001; */ - if (message.channelId !== "") - writer.tag(1001, WireType.LengthDelimited).string(message.channelId); + internalBinaryWrite(message: VisitorData, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string id = 1; */ + if (message.id !== "") + writer.tag(1, WireType.LengthDelimited).string(message.id); + /* int32 timestamp = 5; */ + if (message.timestamp !== 0) + writer.tag(5, WireType.Varint).int32(message.timestamp); let u = options.writeUnknownFields; if (u !== false) (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); @@ -736,9 +709,9 @@ class ChannelAnalytics_Params$Type extends MessageType } } /** - * @generated MessageType for protobuf message youtube.ChannelAnalytics.Params + * @generated MessageType for protobuf message youtube.VisitorData */ -export const ChannelAnalytics_Params = new ChannelAnalytics_Params$Type(); +export const VisitorData = new VisitorData$Type(); // @generated message type with reflection information, may provide speed optimized methods class InnertubePayload$Type extends MessageType { constructor() { @@ -1456,6 +1429,100 @@ class InnertubePayload_VideoThumbnail_Thumbnail$Type extends MessageType { + constructor() { + super("youtube.ChannelAnalytics", [ + { no: 32, name: "params", kind: "message", T: () => ChannelAnalytics_Params } + ]); + } + create(value?: PartialMessage): ChannelAnalytics { + const message = {}; + globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); + if (value !== undefined) + reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics): ChannelAnalytics { + let message = target ?? this.create(), end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* youtube.ChannelAnalytics.Params params */ 32: + message.params = ChannelAnalytics_Params.internalBinaryRead(reader, reader.uint32(), options, message.params); + break; + default: + let u = options.readUnknownField; + if (u === "throw") + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) + (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: ChannelAnalytics, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* youtube.ChannelAnalytics.Params params = 32; */ + if (message.params) + ChannelAnalytics_Params.internalBinaryWrite(message.params, writer.tag(32, WireType.LengthDelimited).fork(), options).join(); + let u = options.writeUnknownFields; + if (u !== false) + (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message youtube.ChannelAnalytics + */ +export const ChannelAnalytics = new ChannelAnalytics$Type(); +// @generated message type with reflection information, may provide speed optimized methods +class ChannelAnalytics_Params$Type extends MessageType { + constructor() { + super("youtube.ChannelAnalytics.Params", [ + { no: 1001, name: "channel_id", kind: "scalar", T: 9 /*ScalarType.STRING*/ } + ]); + } + create(value?: PartialMessage): ChannelAnalytics_Params { + const message = { channelId: "" }; + globalThis.Object.defineProperty(message, MESSAGE_TYPE, { enumerable: false, value: this }); + if (value !== undefined) + reflectionMergePartial(this, message, value); + return message; + } + internalBinaryRead(reader: IBinaryReader, length: number, options: BinaryReadOptions, target?: ChannelAnalytics_Params): ChannelAnalytics_Params { + let message = target ?? this.create(), end = reader.pos + length; + while (reader.pos < end) { + let [fieldNo, wireType] = reader.tag(); + switch (fieldNo) { + case /* string channel_id */ 1001: + message.channelId = reader.string(); + break; + default: + let u = options.readUnknownField; + if (u === "throw") + throw new globalThis.Error(`Unknown field ${fieldNo} (wire type ${wireType}) for ${this.typeName}`); + let d = reader.skip(wireType); + if (u !== false) + (u === true ? UnknownFieldHandler.onRead : u)(this.typeName, message, fieldNo, wireType, d); + } + } + return message; + } + internalBinaryWrite(message: ChannelAnalytics_Params, writer: IBinaryWriter, options: BinaryWriteOptions): IBinaryWriter { + /* string channel_id = 1001; */ + if (message.channelId !== "") + writer.tag(1001, WireType.LengthDelimited).string(message.channelId); + let u = options.writeUnknownFields; + if (u !== false) + (u == true ? UnknownFieldHandler.onWrite : u)(this.typeName, message, writer); + return writer; + } +} +/** + * @generated MessageType for protobuf message youtube.ChannelAnalytics.Params + */ +export const ChannelAnalytics_Params = new ChannelAnalytics_Params$Type(); +// @generated message type with reflection information, may provide speed optimized methods class SoundInfoParams$Type extends MessageType { constructor() { super("youtube.SoundInfoParams", [ diff --git a/src/utils/Constants.ts b/src/utils/Constants.ts index 2dbbf3bbf..767f0c48c 100644 --- a/src/utils/Constants.ts +++ b/src/utils/Constants.ts @@ -35,7 +35,9 @@ export const OAUTH = Object.freeze({ export const CLIENTS = Object.freeze({ WEB: { NAME: 'WEB', - VERSION: '2.20220902.01.00' + VERSION: '2.20230104.01.00', + API_KEY: 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', + API_VERSION: 'v1' }, YTMUSIC: { NAME: 'WEB_REMIX', diff --git a/test/main.test.ts b/test/main.test.ts index 0a86abff7..489ad5693 100644 --- a/test/main.test.ts +++ b/test/main.test.ts @@ -141,6 +141,11 @@ describe('YouTube.js Tests', () => { expect(nop_yt.session.player).toBeUndefined(); }); + it('should create a session from data generated locally', async () => { + const loc_yt = await Innertube.create({ generate_session_locally: true, retrieve_player: false }); + expect(loc_yt.session.context).toBeDefined(); + }); + it('should resolve a URL', async () => { const url = await yt.resolveURL('https://www.youtube.com/@linustechtips'); expect(url.payload.browseId).toBe(CHANNELS[0].ID);