From 0701a2595a3fd089f26504830cc515089dc6f11a Mon Sep 17 00:00:00 2001 From: Shehryar Date: Sat, 27 Feb 2021 23:47:53 +0400 Subject: [PATCH] fixed novel cover and ads script - v2.0.4 --- package.json | 10 +- src/app/pages/novel/novel.component.html | 77 ++------ src/app/services/novel-factory.service.ts | 101 ++++++---- src/app/services/sources/boxnovel.service.ts | 171 ++++++++++++----- src/app/services/sources/novelfull.service.ts | 179 +++++++++++++----- src/app/services/sources/sourceService.ts | 80 +++++--- 6 files changed, 389 insertions(+), 229 deletions(-) diff --git a/package.json b/package.json index f8ea4663..f1e7cdfd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "NovelScraper", - "version": "2.0.3", + "version": "2.0.4", "description": "App for downloading novels from pirate sites.", "homepage": "https://github.com/HanaDigital/NovelScraper", "author": { @@ -39,7 +39,7 @@ }, "devDependencies": { "@angular-builders/custom-webpack": "9.2.0", - "@angular-devkit/build-angular": "0.1000.2", + "@angular-devkit/build-angular": "0.1102.2", "@angular-eslint/builder": "0.0.1-alpha.32", "@angular-eslint/eslint-plugin": "0.0.1-alpha.32", "@angular-eslint/eslint-plugin-template": "0.0.1-alpha.32", @@ -67,7 +67,7 @@ "conventional-changelog-cli": "2.0.34", "core-js": "3.6.5", "cross-env": "7.0.2", - "electron": "^10.1.3", + "electron": "10.4.0", "electron-builder": "22.8.1", "electron-reload": "1.5.0", "eslint": "7.4.0", @@ -86,7 +86,7 @@ "ts-node": "8.10.2", "tslib": "2.0.0", "typescript": "3.9.6", - "wait-on": "5.0.1", + "wait-on": "5.2.1", "webdriver-manager": "12.1.7", "zone.js": "0.10.3" }, @@ -99,7 +99,7 @@ "electron-updater": "4.3.5", "mime": "2.4.6", "nconf": "0.10.0", - "nodepub": "2.2.0", + "nodepub": "3.0.1", "request": "2.88.2", "rimraf": "3.0.2", "shelljs": "0.8.4" diff --git a/src/app/pages/novel/novel.component.html b/src/app/pages/novel/novel.component.html index 9f56c3e0..f997c2c9 100644 --- a/src/app/pages/novel/novel.component.html +++ b/src/app/pages/novel/novel.component.html @@ -2,11 +2,7 @@

- + {{ source.name.toUpperCase() }}

@@ -16,59 +12,30 @@

-
+ }">
{{ database.getPercentage(downloadID) }}
-
+ }">
- +
- - + + - - - - + + + +
@@ -83,22 +50,14 @@

LATEST CHAPTER

{{ novel.latestChapter }}
-
+

TOTAL CHAPTERS

{{ novel.totalChapters }}
-
+

DOWNLOADED

- {{ novel.downloadedChapters }} / - {{ novel.totalChapters }} + {{ novel.downloadedChapters }} / + {{ novel.totalChapters }}

SOURCE

diff --git a/src/app/services/novel-factory.service.ts b/src/app/services/novel-factory.service.ts index 2ddf9981..fa747a1a 100644 --- a/src/app/services/novel-factory.service.ts +++ b/src/app/services/novel-factory.service.ts @@ -1,30 +1,36 @@ -import { Injectable } from '@angular/core'; -import { DatabaseService } from './database.service'; -import { novelObj, chapterObj } from '../resources/types'; -import shellJS from 'shelljs'; // For folder creation, idk why I didn't just do it with fs -import { remote } from 'electron'; -import * as path from 'path'; -import rimraf from 'rimraf'; // For recursive folder deletion -import epubGen from 'nodepub'; -import mime from 'mime'; -import { writeFileSync, createWriteStream, writeFile } from 'fs'; -import request from 'request'; +import { Injectable } from "@angular/core"; +import { DatabaseService } from "./database.service"; +import { novelObj, chapterObj } from "../resources/types"; +import shellJS from "shelljs"; // For folder creation, idk why I didn't just do it with fs +import { remote } from "electron"; +import * as path from "path"; +import rimraf from "rimraf"; // For recursive folder deletion +const epubGen = require("nodepub"); +import mime from "mime"; +import { writeFileSync, createWriteStream, writeFile } from "fs"; +import request from "request"; @Injectable({ - providedIn: 'root' + providedIn: "root", }) export class NovelFactoryService { - // Path to the downloads folder - downloadFolder = path.join(remote.app.getPath("downloads"), "NovelScraper-Library"); + downloadFolder = path.join( + remote.app.getPath("downloads"), + "NovelScraper-Library" + ); - constructor(public database: DatabaseService,) { } + constructor(public database: DatabaseService) {} - async generateEpub(novel: novelObj, chapters: chapterObj[], downloadID: number): Promise { + async generateEpub( + novel: novelObj, + chapters: chapterObj[], + downloadID: number + ): Promise { // Create the folder for storing the novel files - const novelFile = novel.name.replace(/[/\\?%*:|"<>]/g, ''); + const novelFile = novel.name.replace(/[/\\?%*:|"<>]/g, ""); const novelFolder = novel.folderPath; - await shellJS.mkdir('-p', novelFolder); + await shellJS.mkdir("-p", novelFolder); // Save chapters in a json file and set the novel as download await this.saveChapters(novel, chapters, novelFolder); @@ -34,34 +40,41 @@ export class NovelFactoryService { // Set some meta data for the epub file const metadata = { - id: '0000-0000-0001', + id: "0000-0000-0001", + cover: coverPath, title: novel.name, author: novel.author, genre: novel.genre, - language: 'en', - description: 'This novel was downloaded from a pirate site using NovelScraper.' - + ' If possible, please support the author: ' + novel.author, - contents: 'Table of Contents', - images: [] - } + language: "en", + description: + "This novel was downloaded from a pirate site using NovelScraper." + + " If possible, please support the author: " + + novel.author, + contents: "Table of Contents", + images: [], + }; // Generate the epub file and add the chapters - const epub = epubGen.document(metadata, coverPath); - for (const chapter of chapters) epub.addSection(chapter.title, chapter.data); - epub.writeEPUB( - (error) => console.log(error), - novelFolder, novelFile, - () => console.log("[NOVEL FACTORY]: Novel saved!") - ); + const epub = epubGen.document(metadata); + for (const chapter of chapters) + epub.addSection(chapter.title, chapter.data); + + setTimeout(async () => { + await epub.writeEPUB(novelFolder, novelFile); - this.database.updateDownloading(novel.link, false); - this.database.updateDownloaded(novel.link, true); - this.database.updateIsUpdated(novel.link, true); - this.database.setDownloaded(downloadID); + this.database.updateDownloading(novel.link, false); + this.database.updateDownloaded(novel.link, true); + this.database.updateIsUpdated(novel.link, true); + this.database.setDownloaded(downloadID); + }, 2000); } // Save chapters to a json file so we don't re-download when updating - async saveChapters(novel: novelObj, chapters: chapterObj[], novelFolder: string): Promise { + async saveChapters( + novel: novelObj, + chapters: chapterObj[], + novelFolder: string + ): Promise { const chaptersFile = path.join(novelFolder, "chapters.json"); const json = JSON.stringify(chapters, null, 4); writeFile(chaptersFile, json, (err) => { @@ -69,18 +82,24 @@ export class NovelFactoryService { else { // Update total chapters of the novel novel.downloadedChapters = chapters.length; - this.database.updateDownloadedChapters(novel.link, chapters.length); + this.database.updateDownloadedChapters( + novel.link, + chapters.length + ); } - }) + }); } // Download novel cover and return the path - async downloadCover(novelCover: string, novelFolder: string): Promise { + async downloadCover( + novelCover: string, + novelFolder: string + ): Promise { const coverExtension = mime.getExtension(mime.getType(novelCover)); const coverPath = path.join(novelFolder, "cover." + coverExtension); const out = createWriteStream(coverPath); - const req = await request({ method: 'GET', uri: novelCover }); + const req = await request({ method: "GET", uri: novelCover }); await req.pipe(out); return coverPath; diff --git a/src/app/services/sources/boxnovel.service.ts b/src/app/services/sources/boxnovel.service.ts index 57d52122..aa395923 100644 --- a/src/app/services/sources/boxnovel.service.ts +++ b/src/app/services/sources/boxnovel.service.ts @@ -1,26 +1,32 @@ -import { Injectable } from '@angular/core'; -import { chapterObj, novelObj } from 'app/resources/types'; -import { DatabaseService } from '../database.service'; -import { NovelFactoryService } from '../novel-factory.service'; -import { sourceService } from './sourceService'; +import { Injectable } from "@angular/core"; +import { chapterObj, novelObj } from "app/resources/types"; +import { DatabaseService } from "../database.service"; +import { NovelFactoryService } from "../novel-factory.service"; +import { sourceService } from "./sourceService"; @Injectable({ - providedIn: 'root' + providedIn: "root", }) export class BoxnovelService extends sourceService { - - constructor(public database: DatabaseService, public novelFactory: NovelFactoryService) { + constructor( + public database: DatabaseService, + public novelFactory: NovelFactoryService + ) { super(database); } - async searchWIthLink(link: string, source: string, updatingInfo: boolean): Promise { + async searchWIthLink( + link: string, + source: string, + updatingInfo: boolean + ): Promise { this.error = false; this.searching = true; - let novel: novelObj = {}; // Declare novel object + let novel: novelObj = {}; // Declare novel object // Check if the novel exists in the database - novel = this.database.getNovel(link) + novel = this.database.getNovel(link); if (novel && !updatingInfo) { this.sourceNovels.unshift(novel); return novel; @@ -29,7 +35,7 @@ export class BoxnovelService extends sourceService { } try { - const html = await this.getHtml(link); // Get HTML from the link + const html = await this.getHtml(link); // Get HTML from the link // Link if (!updatingInfo) novel.link = link; @@ -38,28 +44,41 @@ export class BoxnovelService extends sourceService { if (!updatingInfo) novel.source = source; // InLibrary - if (!updatingInfo) novel.inLibrary = false; // Set as false to distinguish between novels already present + if (!updatingInfo) novel.inLibrary = false; // Set as false to distinguish between novels already present // Name - const title = html.getElementsByClassName('post-title')[0] - try { title.getElementsByTagName('span')[0].remove(); } catch (error) { console.log("[SERVICE]: Title span warning.") } + const title = html.getElementsByClassName("post-title")[0]; + try { + title.getElementsByTagName("span")[0].remove(); + } catch (error) { + console.log("[SERVICE]: Title span warning."); + } novel.name = title.textContent.trim(); // LatestChapter - novel.latestChapter = html.getElementsByClassName('wp-manga-chapter')[0].getElementsByTagName('a')[0].innerText.trim(); + novel.latestChapter = html + .getElementsByClassName("wp-manga-chapter")[0] + .getElementsByTagName("a")[0] + .innerText.trim(); // Cover - novel.cover = html.getElementsByClassName('summary_image')[0].getElementsByTagName('img')[0].src; + novel.cover = html + .getElementsByClassName("summary_image")[0] + .getElementsByTagName("img")[0].src; // TotalChapters - novel.totalChapters = html.getElementsByClassName('wp-manga-chapter').length; + novel.totalChapters = html.getElementsByClassName( + "wp-manga-chapter" + ).length; // Author(s) - const authorList = html.getElementsByClassName('author-content')[0].getElementsByTagName('a'); + const authorList = html + .getElementsByClassName("author-content")[0] + .getElementsByTagName("a"); try { - let author = "" + let author = ""; for (let i = 0; i < authorList.length; i++) { - author += authorList[i].innerText.trim() + ', '; + author += authorList[i].innerText.trim() + ", "; } novel.author = author.slice(0, -2); } catch (error) { @@ -68,11 +87,13 @@ export class BoxnovelService extends sourceService { } // Genre(s) - const genreList = html.getElementsByClassName('genres-content')[0].getElementsByTagName('a'); + const genreList = html + .getElementsByClassName("genres-content")[0] + .getElementsByTagName("a"); try { let genre = ""; for (let i = 0; i < genreList.length; i++) { - genre = genreList[i].innerText.trim() + ', '; + genre = genreList[i].innerText.trim() + ", "; } novel.genre = genre.slice(0, -2); } catch (error) { @@ -81,9 +102,11 @@ export class BoxnovelService extends sourceService { } // Summary - const summaryList = html.getElementsByClassName('summary__content')[0].getElementsByTagName('p'); + const summaryList = html + .getElementsByClassName("summary__content")[0] + .getElementsByTagName("p"); try { - let summary = "" + let summary = ""; for (let i = 0; i < summaryList.length; i++) { summary += summaryList[i].innerText.trim() + "\n"; } @@ -109,44 +132,59 @@ export class BoxnovelService extends sourceService { this.error = false; this.searching = true; - name = encodeURI(name.replace(/ /g, '+')); // Replace spaces in novel name to a + for creating the search link - const searchLink = "https://boxnovel.com/?s=" + name + "&post_type=wp-manga"; // Search link that will find the novels of this name + name = encodeURI(name.replace(/ /g, "+")); // Replace spaces in novel name to a + for creating the search link + const searchLink = + "https://boxnovel.com/?s=" + name + "&post_type=wp-manga"; // Search link that will find the novels of this name - const foundNovels: novelObj[] = []; // Will store the novels found from this name + const foundNovels: novelObj[] = []; // Will store the novels found from this name try { const html = await this.getHtml(searchLink); let novel: novelObj; - const novelList = html.getElementsByClassName('c-tabs-item__content'); + const novelList = html.getElementsByClassName( + "c-tabs-item__content" + ); for (let i = 0; i < novelList.length; i++) { novel = {}; // Link - novel.link = novelList[i].getElementsByClassName('post-title')[0].getElementsByTagName('a')[0].href; + novel.link = novelList[i] + .getElementsByClassName("post-title")[0] + .getElementsByTagName("a")[0].href; // Source novel.source = source; // Name - novel.name = novelList[i].getElementsByClassName('post-title')[0].getElementsByTagName('a')[0].innerText.trim(); + novel.name = novelList[i] + .getElementsByClassName("post-title")[0] + .getElementsByTagName("a")[0] + .innerText.trim(); // Latest Chapter - novel.latestChapter = novelList[i].getElementsByClassName('chapter')[0].getElementsByTagName('a')[0].innerText.trim(); + novel.latestChapter = novelList[i] + .getElementsByClassName("chapter")[0] + .getElementsByTagName("a")[0] + .innerText.trim(); // Total Chapters novel.totalChapters = 0; // Total chapters don't show when searching for novels // Cover - novel.cover = novelList[i].getElementsByClassName('tab-thumb')[0].getElementsByTagName('img')[0].src; + novel.cover = novelList[i] + .getElementsByClassName("tab-thumb")[0] + .getElementsByTagName("img")[0].src; // Author(s) - const authorList = novelList[i].getElementsByClassName('mg_author')[0].getElementsByTagName('a'); + const authorList = novelList[i] + .getElementsByClassName("mg_author")[0] + .getElementsByTagName("a"); try { let author: string; for (let i = 0; i < authorList.length; i++) { - author = authorList[i].innerText.trim() + ', '; + author = authorList[i].innerText.trim() + ", "; } novel.author = author.slice(0, -2); } catch (error) { @@ -155,11 +193,13 @@ export class BoxnovelService extends sourceService { } // Genre(s) - const genreList = novelList[i].getElementsByClassName('mg_genres')[0].getElementsByTagName('a'); + const genreList = novelList[i] + .getElementsByClassName("mg_genres")[0] + .getElementsByTagName("a"); try { let genre: string; for (let i = 0; i < genreList.length; i++) { - genre = genreList[i].innerText.trim() + ', '; + genre = genreList[i].innerText.trim() + ", "; } novel.genre = genre.slice(0, -2); } catch (error) { @@ -171,7 +211,9 @@ export class BoxnovelService extends sourceService { novel.summary = "N/A"; // Check if novel is already in the searched novel list and remove it - this.sourceNovels = this.sourceNovels.filter(sourceNovel => sourceNovel.link !== novel.link); + this.sourceNovels = this.sourceNovels.filter( + (sourceNovel) => sourceNovel.link !== novel.link + ); // Check if the novel exists in the database const libNovel = this.database.getNovel(novel.link); @@ -183,7 +225,7 @@ export class BoxnovelService extends sourceService { } } } catch (error) { - console.error(error) + console.error(error); this.errorMessage = "ERROR SEARCHING FOR NOVEL"; this.error = true; } @@ -193,19 +235,28 @@ export class BoxnovelService extends sourceService { } async download(novel: novelObj, downloadID: number): Promise { - let downloadedChapters: chapterObj[] = []; // List of download chapters + let downloadedChapters: chapterObj[] = []; // List of download chapters try { const html = await this.getHtml(novel.link); - const chapters = html.getElementsByClassName('wp-manga-chapter'); + const chapters = html.getElementsByClassName("wp-manga-chapter"); // Get chapter links and names let chapterLinks = []; let chapterNames = []; for (let i = 0; i < chapters.length; i++) { - chapterLinks.push(chapters[i].getElementsByTagName('a')[0].getAttribute('href')); - chapterNames.push(chapters[i].getElementsByTagName('a')[0].innerText.trim().replace(/(\r\n|\n|\r)/gm, "")); + chapterLinks.push( + chapters[i] + .getElementsByTagName("a")[0] + .getAttribute("href") + ); + chapterNames.push( + chapters[i] + .getElementsByTagName("a")[0] + .innerText.trim() + .replace(/(\r\n|\n|\r)/gm, "") + ); } chapterLinks.reverse(); chapterNames.reverse(); @@ -215,8 +266,7 @@ export class BoxnovelService extends sourceService { this.database.cancelDownload(downloadID); this.database.updateDownloading(novel.link, false); return; - } - else if (update.startIndex !== 0) { + } else if (update.startIndex !== 0) { downloadedChapters = update.updateChapters; chapterLinks = chapterLinks.slice(update.startIndex); chapterNames = chapterNames.slice(update.startIndex); @@ -226,16 +276,23 @@ export class BoxnovelService extends sourceService { for (let i = 0; i < chapterLinks.length; i++) { if (this.database.isCanceled(downloadID)) { this.database.updateDownloading(novel.link, false); - console.log('Download canceled!') + console.log("Download canceled!"); return; } const html = await this.getHtml(chapterLinks[i]); - const chapterHtml = html.getElementsByClassName('entry-content')[0]; + const chapterHtml = html.getElementsByClassName( + "entry-content" + )[0]; try { - chapterHtml.getElementsByClassName("cha-tit")[0].remove(); // Remove h3 tag from chapter + chapterHtml.getElementsByClassName("cha-tit")[0].remove(); // Remove h3 tag from chapter } catch (error) { - console.log("Missing 'cha-tit' class at chapter index " + i + "and chapter name " + chapterNames[i]); + console.log( + "Missing 'cha-tit' class at chapter index " + + i + + "and chapter name " + + chapterNames[i] + ); } const chapterTitle = chapterNames[i]; @@ -243,12 +300,22 @@ export class BoxnovelService extends sourceService { let chapterBody = "

" + chapterTitle + "

"; chapterBody += chapterHtml.outerHTML; - const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, i, chapterLinks.length); + const chapter = this.prepChapter( + novel, + downloadID, + chapterTitle, + chapterBody, + i, + chapterLinks.length + ); downloadedChapters.push(chapter); } - this.novelFactory.generateEpub(novel, downloadedChapters, downloadID); - + this.novelFactory.generateEpub( + novel, + downloadedChapters, + downloadID + ); } catch (error) { this.database.cancelDownload(downloadID); this.database.updateDownloading(novel.link, false); diff --git a/src/app/services/sources/novelfull.service.ts b/src/app/services/sources/novelfull.service.ts index ed2e424c..61cd7b10 100644 --- a/src/app/services/sources/novelfull.service.ts +++ b/src/app/services/sources/novelfull.service.ts @@ -1,26 +1,32 @@ -import { Injectable } from '@angular/core'; +import { Injectable } from "@angular/core"; import { chapterObj, novelObj } from "app/resources/types"; import { DatabaseService } from "../database.service"; import { NovelFactoryService } from "../novel-factory.service"; import { sourceService } from "./sourceService"; @Injectable({ - providedIn: 'root' + providedIn: "root", }) export class NovelfullService extends sourceService { - - constructor(public database: DatabaseService, public novelFactory: NovelFactoryService) { + constructor( + public database: DatabaseService, + public novelFactory: NovelFactoryService + ) { super(database); } - async searchWIthLink(link: string, source: string, updatingInfo: boolean): Promise { + async searchWIthLink( + link: string, + source: string, + updatingInfo: boolean + ): Promise { this.error = false; this.searching = true; - let novel: novelObj = {}; // Declare novel object + let novel: novelObj = {}; // Declare novel object // Check if the novel exists in the database - novel = this.database.getNovel(link) + novel = this.database.getNovel(link); if (novel && !updatingInfo) { this.sourceNovels.unshift(novel); return novel; @@ -29,7 +35,7 @@ export class NovelfullService extends sourceService { } try { - const html = await this.getHtml(link); // Get HTML from the link + const html = await this.getHtml(link); // Get HTML from the link // Link if (!updatingInfo) novel.link = link; @@ -38,32 +44,54 @@ export class NovelfullService extends sourceService { if (!updatingInfo) novel.source = source; // InLibrary - if (!updatingInfo) novel.inLibrary = false; // Set as false to distinguish between novels already present + if (!updatingInfo) novel.inLibrary = false; // Set as false to distinguish between novels already present //////////////////////// YOUR CODE STARTS HERE /////////////////////////////// // FIXME: Name - novel.name = html.getElementsByClassName('title')[0].textContent; + novel.name = html.getElementsByClassName("title")[0].textContent; // FIXME: LatestChapter - novel.latestChapter = html.getElementsByClassName('l-chapter')[0].getElementsByTagName('a')[0].title; + novel.latestChapter = html + .getElementsByClassName("l-chapter")[0] + .getElementsByTagName("a")[0].title; // FIXME: Cover - novel.cover = "https://novelfull.com" + html.getElementsByClassName('book')[0].getElementsByTagName('img')[0].getAttribute('src'); - console.log(novel.cover) + novel.cover = + "https://novelfull.com" + + html + .getElementsByClassName("book")[0] + .getElementsByTagName("img")[0] + .getAttribute("src"); + console.log(novel.cover); // FIXME: TotalChapters - const lastPage = parseInt(html.getElementsByClassName('pagination')[0].getElementsByClassName('last')[0].getElementsByTagName('a')[0].getAttribute('data-page')); + const lastPage = parseInt( + html + .getElementsByClassName("pagination")[0] + .getElementsByClassName("last")[0] + .getElementsByTagName("a")[0] + .getAttribute("data-page") + ); let totalChapters = lastPage * 50; - const lastPageHtml = await this.getHtml(link + "?page=" + (lastPage + 1) + "&per-page=50"); - totalChapters += lastPageHtml.getElementsByClassName('row')[1].getElementsByTagName('li').length; + const lastPageHtml = await this.getHtml( + link + "?page=" + (lastPage + 1) + "&per-page=50" + ); + totalChapters += lastPageHtml + .getElementsByClassName("row")[1] + .getElementsByTagName("li").length; novel.totalChapters = totalChapters; // FIXME: Author(s) - novel.author = html.getElementsByClassName('info')[0].getElementsByTagName('a')[0].text; + novel.author = html + .getElementsByClassName("info")[0] + .getElementsByTagName("a")[0].text; // FIXME: Genre(s) - const genres = html.getElementsByClassName('info')[0].getElementsByTagName('div')[1].getElementsByTagName('a'); + const genres = html + .getElementsByClassName("info")[0] + .getElementsByTagName("div")[1] + .getElementsByTagName("a"); let genre = ""; for (let i = 0; i < genres.length; i++) { genre += genres[i].innerText + ", "; @@ -71,9 +99,11 @@ export class NovelfullService extends sourceService { novel.genre = genre.slice(0, -2); // FIXME: Summary - const summaryList = html.getElementsByClassName('desc-text')[0].getElementsByTagName('p'); + const summaryList = html + .getElementsByClassName("desc-text")[0] + .getElementsByTagName("p"); try { - let summary = "" + let summary = ""; for (let i = 0; i < summaryList.length; i++) { summary += summaryList[i].innerText.trim() + "\n"; } @@ -103,12 +133,12 @@ export class NovelfullService extends sourceService { //////////////////////// [1] YOUR CODE STARTS HERE /////////////////////////////// // FIXME: Generate the search link from novel name - name = encodeURI(name.replace(/ /g, '+')); // Replace spaces in novel name to a + for creating the search link - const searchLink = "https://novelfull.com/search?keyword=" + name; // Search link that will find the novels of this name + name = encodeURI(name.replace(/ /g, "+")); // Replace spaces in novel name to a + for creating the search link + const searchLink = "https://novelfull.com/search?keyword=" + name; // Search link that will find the novels of this name //////////////////////// YOUR CODE ENDS HERE ///////////////////////////////// - const foundNovels: novelObj[] = []; // Will store the novels found from this name + const foundNovels: novelObj[] = []; // Will store the novels found from this name try { const html = await this.getHtml(searchLink); @@ -116,7 +146,9 @@ export class NovelfullService extends sourceService { //////////////////////// [2] YOUR CODE STARTS HERE /////////////////////////////// // FIXME: Get the list of all search result elements - const novelList = html.getElementsByClassName('list-truyen')[0].getElementsByClassName('row'); + const novelList = html + .getElementsByClassName("list-truyen")[0] + .getElementsByClassName("row"); //////////////////////// YOUR CODE ENDS HERE ///////////////////////////////// @@ -127,25 +159,40 @@ export class NovelfullService extends sourceService { novel.source = source; //////////////////////// [3] YOUR CODE STARTS HERE /////////////////////////////// - console.log(novelList[i]) + console.log(novelList[i]); // FIXME: Link - novel.link = "https://novelfull.com" + novelList[i].getElementsByClassName('truyen-title')[0].getElementsByTagName('a')[0].getAttribute('href'); - console.log(novel.link) + novel.link = + "https://novelfull.com" + + novelList[i] + .getElementsByClassName("truyen-title")[0] + .getElementsByTagName("a")[0] + .getAttribute("href"); + console.log(novel.link); // FIXME: Name - novel.name = novelList[i].getElementsByClassName('truyen-title')[0].getElementsByTagName('a')[0].innerText; + novel.name = novelList[i] + .getElementsByClassName("truyen-title")[0] + .getElementsByTagName("a")[0].innerText; // FIXME: LatestChapter - novel.latestChapter = novelList[i].getElementsByClassName('chapter-text')[0].textContent; + novel.latestChapter = novelList[i].getElementsByClassName( + "chapter-text" + )[0].textContent; // FIXME: Cover - novel.cover = "https://novelfull.com/" + novelList[i].getElementsByTagName('img')[0].getAttribute('src'); + novel.cover = + "https://novelfull.com/" + + novelList[i] + .getElementsByTagName("img")[0] + .getAttribute("src"); // FIXME: TotalChapters - novel.totalChapters = 0; // If totalChapters is unknown, set it to 0 as it will not accept a string + novel.totalChapters = 0; // If totalChapters is unknown, set it to 0 as it will not accept a string // FIXME: Author(s) - novel.author = novelList[i].getElementsByClassName("author")[0].textContent; + novel.author = novelList[i].getElementsByClassName( + "author" + )[0].textContent; // FIXME: Genre(s) novel.genre = "unknown"; @@ -156,7 +203,9 @@ export class NovelfullService extends sourceService { //////////////////////// YOUR CODE ENDS HERE ///////////////////////////////// // Check if novel is already in the searched novel list and remove it - this.sourceNovels = this.sourceNovels.filter(sourceNovel => sourceNovel.link !== novel.link); + this.sourceNovels = this.sourceNovels.filter( + (sourceNovel) => sourceNovel.link !== novel.link + ); // Check if the novel exists in the database const libNovel = this.database.getNovel(novel.link); @@ -171,7 +220,7 @@ export class NovelfullService extends sourceService { novel.source = source; } } catch (error) { - console.error(error) + console.error(error); this.errorMessage = "ERROR SEARCHING FOR NOVEL"; this.error = true; } @@ -181,7 +230,7 @@ export class NovelfullService extends sourceService { } async download(novel: novelObj, downloadID: number): Promise { - let downloadedChapters: chapterObj[] = []; // List of download chapters + let downloadedChapters: chapterObj[] = []; // List of download chapters try { const html = await this.getHtml(novel.link); @@ -190,13 +239,31 @@ export class NovelfullService extends sourceService { let chapterLinks = []; let chapterNames = []; - const lastPage = parseInt(html.getElementsByClassName('pagination')[0].getElementsByClassName('last')[0].getElementsByTagName('a')[0].getAttribute('data-page')) + 1; + const lastPage = + parseInt( + html + .getElementsByClassName("pagination")[0] + .getElementsByClassName("last")[0] + .getElementsByTagName("a")[0] + .getAttribute("data-page") + ) + 1; for (let i = 1; i <= lastPage; i++) { - const currentPageHtml = await this.getHtml(novel.link + "?page=" + i + "&per-page=50"); - const chapters = currentPageHtml.getElementsByClassName('row')[1].getElementsByTagName('li'); + const currentPageHtml = await this.getHtml( + novel.link + "?page=" + i + "&per-page=50" + ); + const chapters = currentPageHtml + .getElementsByClassName("row")[1] + .getElementsByTagName("li"); for (let x = 0; x < chapters.length; x++) { - chapterLinks.push("https://novelfull.com" + chapters[x].getElementsByTagName('a')[0].getAttribute('href')); - chapterNames.push(chapters[x].getElementsByTagName('a')[0].title); + chapterLinks.push( + "https://novelfull.com" + + chapters[x] + .getElementsByTagName("a")[0] + .getAttribute("href") + ); + chapterNames.push( + chapters[x].getElementsByTagName("a")[0].title + ); } } @@ -207,8 +274,7 @@ export class NovelfullService extends sourceService { this.database.cancelDownload(downloadID); this.database.updateDownloading(novel.link, false); return; - } - else if (update.startIndex !== 0) { + } else if (update.startIndex !== 0) { downloadedChapters = update.updateChapters; chapterLinks = chapterLinks.slice(update.startIndex); chapterNames = chapterNames.slice(update.startIndex); @@ -218,7 +284,7 @@ export class NovelfullService extends sourceService { for (let i = 0; i < chapterLinks.length; i++) { if (this.database.isCanceled(downloadID)) { this.database.updateDownloading(novel.link, false); - console.log('Download canceled!') + console.log("Download canceled!"); return; } @@ -228,7 +294,7 @@ export class NovelfullService extends sourceService { // FIXME: you have the html of the chapter page // Get the element that wraps all the paragraphs of the chapter - const chapterHtml = html.getElementsByClassName('chapter-c')[0]; + const chapterHtml = html.getElementsByClassName("chapter-c")[0]; //////////////////////// YOUR CODE ENDS HERE ///////////////////////////////// @@ -237,13 +303,30 @@ export class NovelfullService extends sourceService { let chapterBody = "

" + chapterTitle + "

"; chapterBody += chapterHtml.outerHTML; - - const chapter = this.prepChapter(novel, downloadID, chapterTitle, chapterBody, i, chapterLinks.length); + chapterBody = chapterBody.replace( + /\(adsbygoogle = window.adsbygoogle \|\| \[\]\).push\({}\);/g, + "" + ); + chapterBody = chapterBody.replace(/<\/script>/g, ""); + chapterBody = chapterBody.replace(//g, ""); + console.log(chapterBody); + + const chapter = this.prepChapter( + novel, + downloadID, + chapterTitle, + chapterBody, + i, + chapterLinks.length + ); downloadedChapters.push(chapter); } - this.novelFactory.generateEpub(novel, downloadedChapters, downloadID); - + this.novelFactory.generateEpub( + novel, + downloadedChapters, + downloadID + ); } catch (error) { this.database.cancelDownload(downloadID); this.database.updateDownloading(novel.link, false); diff --git a/src/app/services/sources/sourceService.ts b/src/app/services/sources/sourceService.ts index c3d9d8db..4b50bce9 100644 --- a/src/app/services/sources/sourceService.ts +++ b/src/app/services/sources/sourceService.ts @@ -1,11 +1,10 @@ import { chapterObj, novelObj, update } from "app/resources/types"; import { existsSync, readFileSync } from "fs"; import { DatabaseService } from "../database.service"; -const cloudscraper = (window).require('cloudscraper'); -import * as path from 'path'; +const cloudscraper = (window).require("cloudscraper"); +import * as path from "path"; export class sourceService { - sourceNovels: novelObj[] = []; searching = false; @@ -14,13 +13,19 @@ export class sourceService { scrollPos: number; - constructor(public database: DatabaseService,) { } + constructor(public database: DatabaseService) {} - async searchWIthLink(link: string, source: string, updatingInfo: boolean): Promise { return {}; } + async searchWIthLink( + link: string, + source: string, + updatingInfo: boolean + ): Promise { + return {}; + } - async searchWithName(name: string, source: string): Promise { } + async searchWithName(name: string, source: string): Promise {} - async download(novel: novelObj, downloadID: number): Promise { } + async download(novel: novelObj, downloadID: number): Promise {} update(novel: novelObj, numOfChapters: number): update { if (numOfChapters > novel.totalChapters) { @@ -28,16 +33,16 @@ export class sourceService { this.database.updateTotalChapters(novel.link, numOfChapters); } - const updateFile = path.join(novel.folderPath, 'chapters.json'); + const updateFile = path.join(novel.folderPath, "chapters.json"); if (!existsSync(updateFile)) { novel.downloaded = false; this.database.updateDownloaded(novel.link, false); - console.log("[SERVICE]: Missing update file. Downloading again!") + console.log("[SERVICE]: Missing update file. Downloading again!"); return { updateChapters: undefined, startIndex: 0 }; } - const json: string = readFileSync(updateFile, 'utf8'); + const json: string = readFileSync(updateFile, "utf8"); const updateChapters: chapterObj[] = JSON.parse(json); if (numOfChapters === novel.downloadedChapters) { @@ -45,13 +50,18 @@ export class sourceService { return { updateChapters: undefined, startIndex: -1 }; } - return { updateChapters: updateChapters, startIndex: updateChapters.length } + return { + updateChapters: updateChapters, + startIndex: updateChapters.length, + }; } pushOrUpdateNovel(novel: novelObj, updatingInfo: boolean): void { if (!novel.downloadedChapters) novel.downloadedChapters = 0; if (!updatingInfo) { - this.sourceNovels = this.sourceNovels.filter(sourceNovel => sourceNovel.link !== novel.link); + this.sourceNovels = this.sourceNovels.filter( + (sourceNovel) => sourceNovel.link !== novel.link + ); this.sourceNovels.unshift(novel); } else if (novel.totalChapters > novel.downloadedChapters) { this.database.updateNovelObj(novel); @@ -59,35 +69,57 @@ export class sourceService { } } - prepChapter(novel: novelObj, downloadID: number, title: string, chapterBody: string, currentPos: number, destPos: number): chapterObj { + prepChapter( + novel: novelObj, + downloadID: number, + title: string, + chapterBody: string, + currentPos: number, + destPos: number + ): chapterObj { chapterBody = chapterBody.replace(//gi, ""); // Remove any script tags usually used with ads (Security+) chapterBody = chapterBody.replace(//gi, ""); // Remove any iframes usually used with ads chapterBody = chapterBody.replace(/< *br *>/gi, "
"); // Make sure all
tags end correctly for xhtml - chapterBody = chapterBody.replace(/
/gi, ""); // Remove any useless
tags - chapterBody = chapterBody.replace(/(])*)>/gi, "$1/>"); // Make sure img tag ends correctly for xhtml + chapterBody = chapterBody.replace(/
/gi, ""); // Remove any useless
tags + chapterBody = chapterBody.replace( + /(])*)>/gi, + "$1/>" + ); // Make sure img tag ends correctly for xhtml chapterBody += this.addPropoganda(novel.author); - const percentage = (((currentPos / destPos) * 100).toFixed(2)); + const percentage = ((currentPos / destPos) * 100).toFixed(2); this.database.updateDownloadTracker(downloadID, percentage); console.log(percentage); - return { title: title, data: chapterBody, read: false, scroll: 0 } as chapterObj; + return { + title: title, + data: chapterBody, + read: false, + scroll: 0, + } as chapterObj; } async getHtml(link: string): Promise { - const stringHtml: string = await cloudscraper({ method: 'GET', url: link, }, (error, response, novelHtmlString) => { - return novelHtmlString; - }); - - const html = document.createElement('html'); + const stringHtml: string = await cloudscraper( + { method: "GET", url: link }, + (error, response, novelHtmlString) => { + return novelHtmlString; + } + ); + + const html = document.createElement("html"); html.innerHTML = stringHtml; return html; } addPropoganda(author: string): string { - return "

" + + return ( + "

" + "

This novel was scraped from a pirate site using NovelScraper.

" + - "

If possible, please support the author(s) of this novel: " + author + "

"; + "

If possible, please support the author(s) of this novel: " + + author + + "

" + ); } }