fix(fetch): Add capacity to change UA string

src/sources/en/novelupdates.js

@@ -1,4 +1,5 @@
 import * as cheerio from 'cheerio';
+import { fetchApi, fetchHtml, cloudflareCheck } from '@utils/fetch/fetch';
 import { defaultTo } from 'lodash-es';
 import { FilterInputs } from '../types/filterTypes';
 const sourceId = 50;
@@ -7,10 +8,8 @@ const sourceName = 'Novel Updates';
 
 const baseUrl = 'https://www.novelupdates.com/';
 
-let headers = new Headers({
-  'User-Agent':
-    "'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
-});
+const userAgent =
+  'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36';
 
 const getPopularNovelsUrl = (page, { showLatestNovels, filters }) => {
   let url = `${baseUrl}${
@@ -55,11 +54,11 @@ const getPopularNovelsUrl = (page, { showLatestNovels, filters }) => {
 const popularNovels = async (page, { showLatestNovels, filters }) => {
   const url = getPopularNovelsUrl(page, { showLatestNovels, filters });
 
-  const result = await fetch(url, {
-    method: 'GET',
-    headers: headers,
+  const body = await fetchHtml({
+    url,
+    sourceId,
+    init: { headers: { 'User-Agent': userAgent } },
   });
-  const body = await result.text();
 
   const loadedCheerio = cheerio.load(body);
 
@@ -89,11 +88,11 @@ const popularNovels = async (page, { showLatestNovels, filters }) => {
 const parseNovelAndChapters = async novelUrl => {
   const url = `${baseUrl}series/${novelUrl}`;
 
-  const result = await fetch(url, {
-    method: 'GET',
-    headers: headers,
+  const body = await fetchHtml({
+    url,
+    sourceId,
+    init: { headers: { 'User-Agent': userAgent } },
   });
-  const body = await result.text();
 
   let loadedCheerio = cheerio.load(body);
 
@@ -133,14 +132,15 @@ const parseNovelAndChapters = async novelUrl => {
   formData.append('mygrr', 0);
   formData.append('mypostid', parseInt(novelId, 10));
 
-  const data = await fetch(
-    'https://www.novelupdates.com/wp-admin/admin-ajax.php',
-    {
+  const data = await fetchApi({
+    url: 'https://www.novelupdates.com/wp-admin/admin-ajax.php',
+    init: {
       method: 'POST',
-      headers,
       body: formData,
+      headers: { 'User-Agent': userAgent },
     },
-  );
+    sourceId,
+  });
   const text = await data.text();
 
   loadedCheerio = cheerio.load(text);
@@ -179,11 +179,16 @@ const parseChapter = async (novelUrl, chapterUrl) => {
 
   let chapterText = '';
 
-  result = await fetch(url, {
-    method: 'GET',
-    headers: headers,
+  result = await fetchApi({
+    url,
+    init: {
+      method: 'GET',
+      headers: { 'User-Agent': userAgent },
+    },
+    sourceId,
   });
   body = await result.text();
+  cloudflareCheck(body);
 
   // console.log(result.url);
 
@@ -318,11 +323,11 @@ const searchNovels = async searchTerm => {
   const url =
     'https://www.novelupdates.com/?s=' + searchTerm + '&post_type=seriesplans';
 
-  const res = await fetch(url, {
-    method: 'GET',
-    headers: headers,
+  const body = await fetchHtml({
+    url,
+    sourceId,
+    init: { headers: { 'User-Agent': userAgent } },
   });
-  const body = await res.text();
 
   const loadedCheerio = cheerio.load(body);
 

src/utils/fetch/fetch.ts

@@ -1,23 +1,41 @@
 import { getSourceStorage } from '@hooks/useSourceStorage';
 
 export const defaultUserAgentString =
-  'Mozilla/5.0 (Linux; Android 13) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/108.0.5359.128 Mobile Safari/537.36';
+  'Mozilla/5.0 (Linux; Android 10; K) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/113.0.0.0 Mobile Safari/537.36';
 
 interface FetchParams {
-  url: string;
-  init?: RequestInit;
-  sourceId?: number;
+  url: string; // URL of request
+  init?: RequestInit; // Variable for passing headers and other information
+  sourceId?: number; // ID number of source for cookies
 }
 
+// Checks if we bypassed cloudflare. If we failed to bypass, throw error.
+export const cloudflareCheck = (text: string) => {
+  if (text.length > 0) {
+    if (
+      text.includes('Enable JavaScript and cookies to continue') ||
+      text.includes('Checking if the site connection is secure')
+    ) {
+      throw Error(
+        "The app couldn't bypass the source's Cloudflare protection.\n\nOpen the source in WebView to bypass the Cloudflare protection.",
+      );
+    }
+  }
+};
+
 export const fetchApi = async ({
   url,
   init,
   sourceId,
 }: FetchParams): Promise<Response> => {
-  const headers = new Headers({
-    ...init?.headers,
+  let headers = new Headers({
     'User-Agent': defaultUserAgentString,
+    ...init?.headers,
   });
+  // 'User-Agent' can be overwritten by defining
+  // init: { headers: { 'User-Agent': 'New user agent!' } },
+  // You can have NO user agent by doing this:
+  // init: { headers: { 'User-Agent': undefined } },
 
   if (sourceId) {
     const { cookies = '' } = getSourceStorage(sourceId);
@@ -34,14 +52,7 @@ export const fetchHtml = async (params: FetchParams): Promise<string> => {
   const res = await fetchApi(params);
   const html = await res.text();
 
-  if (
-    html.includes('Enable JavaScript and cookies to continue') ||
-    html.includes('Checking if the site connection is secure')
-  ) {
-    throw Error(
-      "The app couldn't bypass the source's Cloudflare protection.\n\nOpen the source in WebView to bypass the Cloudflare protection.",
-    );
-  }
+  cloudflareCheck(html);
 
   return html;
 };