Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added source Bookriver #641

Merged
merged 3 commits into from
May 29, 2023
Merged

Added source Bookriver #641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
7e0c5a9
add BookRiver
Rider21 May 21, 2023
e9ded6e
Update bookriver.js
Rider21 May 22, 2023
d5e7a98
updated tags, genres
Rider21 May 22, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7,878 changes: 1,586 additions & 6,292 deletions src/sources/multisrc/rulate/RulateGenerator.js
View file
Open in desktop

Large diffs are not rendered by default.

9 changes: 4 additions & 5 deletions src/sources/multisrc/rulate/RulateScraper.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { FilterInputs } from '../../types/filterTypes';
import { htmlToText } from '../../helpers/htmlToText';
import { Status } from '../../helpers/constants';
import * as cheerio from 'cheerio';
import { defaultTo } from 'lodash-es';

class RulateScraper {
constructor(sourceId, baseUrl, sourceName, filter) {
Expand Down Expand Up @@ -83,10 +82,10 @@ class RulateScraper {
const baseUrl = this.baseUrl;
const sourceId = this.sourceId;
let url = baseUrl + '/search?t=&cat=2';
url += '&sort=' + defaultTo(filters?.sort, showLatestNovels ? '4' : '6');
url += '&type=' + defaultTo(filters?.type, '0');
url += '&atmosphere=' + defaultTo(filters?.atmosphere, '0');
url += '&adult=' + defaultTo(filters?.adult, '0');
url += '&sort=' + showLatestNovels ? '4' : filters?.sort || '6';
url += '&type=' + filters?.type || '0';
url += '&atmosphere=' + filters?.atmosphere || '0';
url += '&adult=' + filters?.adult || '0';

if (filters?.genres?.length) {
url += filters.genres.map(i => `&genres[]=${i}`).join('');
Expand Down
37 changes: 19 additions & 18 deletions src/sources/ru/authortoday.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ import { Status, defaultCoverUri } from '../helpers/constants';
import { htmlToText } from '../helpers/htmlToText';
import { FilterInputs } from '../types/filterTypes';
import * as cheerio from 'cheerio';
import { defaultTo } from 'lodash-es';
import dayjs from 'dayjs';

const sourceId = 142;
Expand All @@ -21,14 +20,13 @@ const popularNovels = async (page, { showLatestNovels, filters }) => {
}

url +=
'&sorting=' +
defaultTo(filters?.sort, showLatestNovels ? 'recent' : 'popular');
'&sorting=' + (showLatestNovels ? 'recent' : filters?.sort || 'popular');

url += '&form=' + defaultTo(filters?.form, 'any');
url += '&state=' + defaultTo(filters?.state, 'any');
url += '&series=' + defaultTo(filters?.series, 'any');
url += '&access=' + defaultTo(filters?.access, 'any');
url += '&promo=' + defaultTo(filters?.promo, 'hide');
url += '&form=' + (filters?.form || 'any');
url += '&state=' + (filters?.state || 'any');
url += '&series=' + (filters?.series || 'any');
url += '&access=' + (filters?.access || 'any');
url += '&promo=' + (filters?.promo || 'hide');

const result = await fetch(url, {
headers: {
Expand All @@ -41,14 +39,18 @@ const popularNovels = async (page, { showLatestNovels, filters }) => {
return { novels: [] };
}

let novels = json.searchResults.map(novel => ({
sourceId,
novelName: novel.title,
novelCover: novel?.coverUrl
? 'https://cm.author.today/content/' + novel.coverUrl
: defaultCoverUri,
novelUrl: Math.floor(novel.id).toString(),
}));
let novels = [];

json.searchResults.forEach(novel =>
novels.push({
sourceId,
novelName: novel.title,
novelCover: novel?.coverUrl
? 'https://cm.author.today/content/' + novel.coverUrl
: defaultCoverUri,
novelUrl: Math.floor(novel.id).toString(),
}),
);

return { novels };
};
Expand Down Expand Up @@ -233,6 +235,7 @@ const filters = [
{ label: 'Любовное фэнтези', value: 'love-fantasy' },
{ label: 'Любовные романы', value: 'romance' },
{ label: 'Мистика', value: 'paranormal' },
{ label: 'Назад в СССР', value: 'back-to-ussr' },
{ label: 'Научная фантастика', value: 'science-fiction' },
{ label: 'Подростковая проза', value: 'teen-prose' },
{ label: 'Политический роман', value: 'political-fiction' },
Expand All @@ -252,7 +255,6 @@ const filters = [
{ label: 'РеалРПГ', value: 'realrpg' },
{ label: 'Романтическая эротика', value: 'romantic-erotika' },
{ label: 'Сказка', value: 'fairy-tale' },
{ label: 'Слэш', value: 'slash' },
{ label: 'Современная проза', value: 'modern-prose' },
{ label: 'Современный любовный роман', value: 'contemporary-romance' },
{ label: 'Социальная фантастика', value: 'sf-social' },
Expand All @@ -263,7 +265,6 @@ const filters = [
{ label: 'Фантастика', value: 'sci-fi' },
{ label: 'Фантастический детектив', value: 'detective-science-fiction' },
{ label: 'Фанфик', value: 'fanfiction' },
{ label: 'Фемслэш', value: 'femslesh' },
{ label: 'Фэнтези', value: 'fantasy' },
{ label: 'Шпионский детектив', value: 'spy-mystery' },
{ label: 'Эпическое фэнтези', value: 'epic-fantasy' },
Expand Down
198 changes: 198 additions & 0 deletions src/sources/ru/bookriver.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,198 @@
import dayjs from 'dayjs';
import * as cheerio from 'cheerio';
import { Status } from '../helpers/constants';
import { FilterInputs } from '../types/filterTypes';

const sourceId = 164;
const sourceName = 'BookRiver';
const baseUrl = 'https://bookriver.ru';

const popularNovels = async (page, { showLatestNovels, filters }) => {
let url = baseUrl + `/genre?page=${page}&perPage=24&sortingType=`;
url += showLatestNovels ? 'last-update' : filters?.sort || 'bestseller';

if (filters?.genres?.length) {
url += '&g=' + filters.genres.join(',');
}

const result = await fetch(url);
const body = await result.text();

const loadedCheerio = cheerio.load(body);
let json = loadedCheerio('#__NEXT_DATA__').html();
json = JSON.parse(json);

let novels = [];
json.props.pageProps.state.pagesFilter.genre.books.forEach(novel =>
novels.push({
sourceId,
novelName: novel.name,
novelCover: novel.coverImages[0].url,
novelUrl: baseUrl + '/book/' + novel.slug,
}),
);

return { novels };
};

const parseNovelAndChapters = async novelUrl => {
const result = await fetch(novelUrl);
const body = await result.text();

const loadedCheerio = cheerio.load(body);
const json = loadedCheerio('#__NEXT_DATA__').html();
const book = JSON.parse(json).props.pageProps.state.book.bookPage;

let novel = {
sourceId,
sourceName,
url: novelUrl,
novelUrl,
novelName: book.name,
novelCover: book.coverImages[0].url,
summary: book.annotation,
author: book.author.name,
genre: book.tags.map(item => item.name).join(','),
status:
book.statusComplete === 'writing' ? Status.ONGOING : Status.COMPLETED,
};

let chapters = [];

book.ebook.chapters.forEach(chapter => {
if (chapter.available) {
chapters.push({
chapterName: chapter.name,
releaseDate: dayjs(
chapter?.firstPublishedAt || chapter.createdAt,
).format('LLL'),
chapterUrl: baseUrl + '/reader/' + book.slug + '/' + chapter.chapterId,
});
}
});

novel.chapters = chapters;
return novel;
};

const parseChapter = async (novelUrl, chapterUrl) => {
const url = 'https://api.bookriver.ru/api/v1/books/chapter/text/';
const result = await fetch(url + chapterUrl.split('/').pop());
const json = await result.json();

const chapter = {
sourceId,
novelUrl,
chapterUrl,
chapterName: json.data.name,
chapterText: json.data.content,
};
return chapter;
};

const searchNovels = async searchTerm => {
const url = `${baseUrl}/search/books?keyword=${searchTerm}`;
const result = await fetch(url);
const body = await result.text();

const loadedCheerio = cheerio.load(body);
let json = loadedCheerio('#__NEXT_DATA__').html();
json = JSON.parse(json);

let novels = [];
json.props.pageProps.state.catalog.books.books.forEach(novel =>
novels.push({
sourceId,
novelName: novel.name,
novelCover: novel.coverImages[0].url,
novelUrl: baseUrl + '/book/' + novel.slug,
}),
);

return novels;
};

const filters = [
{
key: 'sort',
label: 'Сортировка',
values: [
{ label: 'Бестселлеры', value: 'bestseller' },
{ label: 'Дате добавления', value: 'newest' },
{ label: 'Дате обновления', value: 'last-update' },
],
inputType: FilterInputs.Picker,
},
{
key: 'genres',
label: 'жанры',
values: [
{ label: 'Альтернативная история', value: 'alternativnaya-istoriya' },
{ label: 'Боевая фантастика', value: 'boevaya-fantastika' },
{ label: 'Боевое фэнтези', value: 'boevoe-fentezi' },
{ label: 'Бытовое фэнтези', value: 'bytovoe-fentezi' },
{ label: 'Героическая фантастика', value: 'geroicheskaya-fantastika' },
{ label: 'Героическое фэнтези', value: 'geroicheskoe-fentezi' },
{ label: 'Городское фэнтези', value: 'gorodskoe-fentezi' },
{ label: 'Детектив', value: 'detektiv' },
{ label: 'Детективная фантастика', value: 'detektivnaya-fantastika' },
{ label: 'Жёсткая эротика', value: 'zhyostkaya-erotika' },
{ label: 'Исторический детектив', value: 'istoricheskii-detektiv' },
{
label: 'Исторический любовный роман',
value: 'istoricheskii-lyubovnyi-roman',
},
{ label: 'Историческое фэнтези', value: 'istoricheskoe-fentezi' },
{ label: 'Киберпанк', value: 'kiberpank' },
{ label: 'Классический детектив', value: 'klassicheskii-detektiv' },
{ label: 'Короткий любовный роман', value: 'korotkii-lyubovnyi-roman' },
{ label: 'Космическая фантастика', value: 'kosmicheskaya-fantastika' },
{ label: 'Криминальный детектив', value: 'kriminalnyi-detektiv' },
{ label: 'ЛитРПГ', value: 'litrpg' },
{ label: 'Любовная фантастика', value: 'lyubovnaya-fantastika' },
{ label: 'Любовное фэнтези', value: 'lyubovnoe-fentezi' },
{ label: 'Любовный роман', value: 'lyubovnyi-roman' },
{ label: 'Мистика', value: 'mistika' },
{ label: 'Молодежная проза', value: 'molodezhnaya-proza' },
{ label: 'Научная фантастика', value: 'nauchnaya-fantastika' },
{
label: 'Остросюжетный любовный роман',
value: 'ostrosyuzhetnyi-lyubovnyi-roman',
},
{ label: 'Политический детектив', value: 'politicheskii-detektiv' },
{ label: 'Попаданцы', value: 'popadantsy' },
{ label: 'Постапокалипсис', value: 'postapokalipsis' },
{ label: 'Приключенческое фэнтези', value: 'priklyuchencheskoe-fentezi' },
{ label: 'Романтическая эротика', value: 'romanticheskaya-erotika' },
{ label: 'С элементами эротики', value: 's-elementami-erotiki' },
{ label: 'Славянское фэнтези', value: 'slavyanskoe-fentezi' },
{
label: 'Современный любовный роман',
value: 'sovremennyi-lyubovnyi-roman',
},
{ label: 'Социальная фантастика', value: 'sotsialnaya-fantastika' },
{ label: 'Тёмное фэнтези', value: 'temnoe-fentezi' },
{ label: 'Фантастика', value: 'fantastika' },
{ label: 'Фэнтези', value: 'fentezi' },
{ label: 'Шпионский детектив', value: 'shpionskii-detektiv' },
{ label: 'Эпическое фэнтези', value: 'epicheskoe-fentezi' },
{ label: 'Эротика', value: 'erotika' },
{ label: 'Эротическая фантастика', value: 'eroticheskaya-fantastika' },
{ label: 'Эротический фанфик', value: 'eroticheskii-fanfik' },
{ label: 'Эротическое фэнтези', value: 'eroticheskoe-fentezi' },
{ label: 'Юмористический детектив', value: 'yumoristicheskii-detektiv' },
{ label: 'Юмористическое фэнтези', value: 'yumoristicheskoe-fentezi' },
],
inputType: FilterInputs.Checkbox,
},
];

const BookRiverScraper = {
popularNovels,
parseNovelAndChapters,
parseChapter,
searchNovels,
filters,
};

export default BookRiverScraper;
3 changes: 1 addition & 2 deletions src/sources/ru/ficbook.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import { Status, defaultCoverUri } from '../helpers/constants';
import { FilterInputs } from '../types/filterTypes';
import * as cheerio from 'cheerio';
import { defaultTo } from 'lodash-es';

const sourceId = 139;
const sourceName = 'Книга Фанфиков';
Expand All @@ -14,7 +13,7 @@ const popularNovels = async (page, { filters }) => {
if (filters?.directions) {
url += '/popular/' + filters.directions;
} else {
url += `/${defaultTo(filters?.sort, 'fanfiction')}?p=${page}`;
url += '/' + (filters?.sort || 'fanfiction') + '?p=' + page;
}

const result = await fetch(url);
Expand Down
28 changes: 14 additions & 14 deletions src/sources/ru/jaomix.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import * as cheerio from 'cheerio';
import { defaultTo } from 'lodash-es';
import { Status } from '../helpers/constants';
import { FilterInputs } from '../types/filterTypes';

Expand All @@ -10,7 +9,7 @@ const baseUrl = 'https://jaomix.ru';

const popularNovels = async (page, { showLatestNovels, filters }) => {
let url = baseUrl + '/?searchrn&sortby=';
url += defaultTo(filters?.sort, showLatestNovels ? 'upd' : 'count');
url += showLatestNovels ? 'upd' : filters?.sort || 'count';

if (filters?.type?.length) {
url += filters.type.map(i => `&lang[]=${i}`).join('');
Expand Down Expand Up @@ -162,17 +161,6 @@ const filters = [
key: 'genres',
label: 'Жанры',
values: [
{ label: 'Adult', value: 'Adult' },
{ label: 'Ecchi', value: 'Ecchi' },
{ label: 'Josei', value: 'Josei' },
{ label: 'Lolicon', value: 'Lolicon' },
{ label: 'Mature', value: 'Mature' },
{ label: 'Sci-fi', value: 'Sci-fi' },
{ label: 'Shoujo', value: 'Shoujo' },
{ label: 'Wuxia', value: 'Wuxia' },
{ label: 'Xianxia', value: 'Xianxia' },
{ label: 'Xuanhuan', value: 'Xuanhuan' },
{ label: 'Yaoi', value: 'Yaoi' },
{ label: 'Боевые Искусства', value: 'Боевые Искусства' },
{ label: 'Виртуальный Мир', value: 'Виртуальный Мир' },
{ label: 'Гарем', value: 'Гарем' },
Expand All @@ -182,6 +170,7 @@ const filters = [
{ label: 'Истории из жизни', value: 'Истории из жизни' },
{ label: 'Исторический', value: 'Исторический' },
{ label: 'История', value: 'История' },
{ label: 'Исэкай', value: 'Исэкай' },
{ label: 'Комедия', value: 'Комедия' },
{ label: 'Меха', value: 'Меха' },
{ label: 'Мистика', value: 'Мистика' },
Expand All @@ -191,9 +180,9 @@ const filters = [
{ label: 'Приключения', value: 'Приключения' },
{ label: 'Психология', value: 'Психология' },
{ label: 'Романтика', value: 'Романтика' },
{ label: 'Сверхъестественное', value: 'Сверхъестественное' },
{ label: 'Сёнэн', value: 'Сёнэн' },
{ label: 'Сёнэн-ай', value: 'Сёнэн-ай' },
{ label: 'Сверхъестественное', value: 'Сверхъестественное' },
{ label: 'Спорт', value: 'Спорт' },
{ label: 'Сэйнэн', value: 'Сэйнэн' },
{ label: 'Сюаньхуа', value: 'Сюаньхуа' },
Expand All @@ -206,6 +195,17 @@ const filters = [
{ label: 'Шоунен', value: 'Шоунен' },
{ label: 'Экшн', value: 'Экшн' },
{ label: 'Этти', value: 'Этти' },
{ label: 'Юри', value: 'Юри' },
{ label: 'Adult', value: 'Adult' },
{ label: 'Ecchi', value: 'Ecchi' },
{ label: 'Josei', value: 'Josei' },
{ label: 'Lolicon', value: 'Lolicon' },
{ label: 'Mature', value: 'Mature' },
{ label: 'Shoujo', value: 'Shoujo' },
{ label: 'Wuxia', value: 'Wuxia' },
{ label: 'Xianxia', value: 'Xianxia' },
{ label: 'Xuanhuan', value: 'Xuanhuan' },
{ label: 'Yaoi', value: 'Yaoi' },
],
inputType: FilterInputs.Checkbox,
},
Expand Down
Loading