diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index b034619be6124d..77026d75ece61f 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,8 +1,8 @@ // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/devcontainers/images/blob/v0.2.24/src/javascript-node/.devcontainer/devcontainer.json +// https://github.com/devcontainers/images/blob/v0.3.24/src/javascript-node/.devcontainer/devcontainer.json { "name": "Node.js", - "image": "mcr.microsoft.com/devcontainers/javascript-node:18-bullseye", + "image": "mcr.microsoft.com/devcontainers/javascript-node:22-bookworm", // Configure tool-specific properties. "customizations": { @@ -16,8 +16,9 @@ "EditorConfig.EditorConfig", "esbenp.prettier-vscode", "deepscan.vscode-deepscan", - "rangav.vscode-thunder-client", "SonarSource.sonarlint-vscode", + "unifiedjs.vscode-mdx", + "VASubasRaj.flashpost", // Thunder Client is paywalled in WSL/Codespaces/SSH > 2.30.0 "ZihanLi.at-helper" ] } @@ -37,12 +38,12 @@ } }, - "onCreateCommand": "sudo apt-get update && export DEBIAN_FRONTEND=noninteractive && sudo apt-get -y install --no-install-recommends ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-utils redis-server && sudo apt-get autoremove -y && sudo apt-get clean -y && sudo rm -rf /var/lib/apt/lists/*", + "onCreateCommand": "sudo apt-get update && export DEBIAN_FRONTEND=noninteractive && sudo apt-get -y install --no-install-recommends ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-utils redis-server default-jre-headless && sudo apt-get autoremove -y && sudo apt-get clean -y && sudo rm -rf /var/lib/apt/lists/*", - "updateContentCommand": "pnpm i && pnpm i -C website && pnpm rb", + "updateContentCommand": "export JAVA_HOME=/usr/lib/jvm/default-java && pnpm config set store-dir ~/.local/share/pnpm/store && pnpm i && pnpm rb", // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "pnpm i && pnpm i -C website && pnpm rb", + "postCreateCommand": "pnpm i && pnpm rb", // Disable auto start dev env since codespaces sometimes fails to attach to the terminal // "postAttachCommand": { diff --git a/.dockerignore b/.dockerignore index 7abe42543ba1a3..81d65ee79f4eb1 100644 --- a/.dockerignore +++ b/.dockerignore @@ -8,9 +8,7 @@ Dockerfile* LICENSE Procfile app-minimal -assets coverage -docs node_modules test @@ -27,21 +25,21 @@ test .(yarn|npm|nvm)rc *.md app.json +eslint.config.mjs docker-compose* fly.toml jsconfig.json npm-debug.log process.json package-lock.json +vitest.config.ts vercel.json -#git but keep the git commit hash +# git but keep the git commit hash .git/logs -.git/objects .git/index .git/info .git/hooks -#rsshub auxiliary files -lib/radar-rules.js -lib/v2/**/radar.js +# rsshub auxiliary files +lib/routes/**/radar.js diff --git a/.eslintignore b/.eslintignore deleted file mode 100644 index b28531c34a1101..00000000000000 --- a/.eslintignore +++ /dev/null @@ -1,6 +0,0 @@ -coverage -.vscode -docker-compose.yml -!/.github -!/docs/.vuepress -website diff --git a/.eslintrc.json b/.eslintrc.json deleted file mode 100644 index 7a65cd5d6bcc7d..00000000000000 --- a/.eslintrc.json +++ /dev/null @@ -1,113 +0,0 @@ -{ - "extends": ["eslint:recommended", "plugin:n/recommended", "plugin:prettier/recommended", "plugin:yml/recommended"], - "plugins": ["prettier"], - "parserOptions": { - "ecmaVersion": "latest", - "sourceType": "module" - }, - "env": { - "node": true, - "es6": true, - "browser": true - }, - "rules": { - // possible problems - "array-callback-return": 2, - "no-await-in-loop": 2, - "no-control-regex": 0, - "no-duplicate-imports": 2, - "no-prototype-builtins": 0, - "no-unsafe-negation": 2, - "require-atomic-updates": 0, - // suggestions - "arrow-body-style": 2, - "block-scoped-var": 2, - "curly": 2, - "dot-notation": 2, - "eqeqeq": 2, - "no-console": 2, - "no-eval": 2, - "no-extend-native": 2, - "no-extra-label": 2, - "no-global-assign": 2, - "no-implicit-coercion": [ - "error", - { - "boolean": false, - "number": false, - "string": false, - "disallowTemplateShorthand": true - } - ], - "no-implicit-globals": 2, - "no-labels": 2, - "no-multi-str": 2, - "no-new-func": 2, - "no-restricted-imports": 2, - "no-unneeded-ternary": 2, - "no-useless-computed-key": 2, - "no-useless-concat": 1, - "no-useless-rename": 2, - "no-var": 2, - "object-shorthand": 2, - "prefer-arrow-callback": 2, - "prefer-const": 2, - "prefer-regex-literals": 1, - "require-await": 2, - "spaced-comment": 2, - // layout & formatting - "arrow-parens": 2, - "arrow-spacing": 2, - "comma-spacing": 2, - "comma-style": 2, - "func-call-spacing": 2, - "keyword-spacing": 2, - "linebreak-style": 2, - "lines-around-comment": 2, - "no-multiple-empty-lines": 2, - "no-trailing-spaces": 2, - "rest-spread-spacing": 2, - "semi": 2, - "space-before-blocks": 2, - "space-in-parens": 2, - "space-infix-ops": 2, - "space-unary-ops": 2, - // plugin specific - "n/no-extraneous-require": [ - "error", - { - "allowModules": ["puppeteer-extra-plugin-user-preferences", "puppeteer-extra-plugin-user-data-dir"] - } - ], - "n/no-deprecated-api": 1, - "n/no-missing-require": 0, - "n/no-process-exit": 0, - "n/no-unpublished-require": [ - "error", - { - "allowModules": ["tosource"] - } - ], - "prettier/prettier": 0, - "yml/quotes": [ - "error", - { - "prefer": "single" - } - ] - }, - "overrides": [ - { - "files": ["*.yaml", "*.yml"], - "parser": "yaml-eslint-parser", - "rules": { - "lines-around-comment": [ - "error", - { - "beforeBlockComment": false - } - ] - } - } - ] -} diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml deleted file mode 100644 index 1642f7a25db3f7..00000000000000 --- a/.github/FUNDING.yml +++ /dev/null @@ -1,5 +0,0 @@ -# These are supported funding model platforms -github: DIYgod -open_collective: RSSHub -patreon: DIYgod -custom: ['https://afdian.net/@diygod', 'https://archive.diygod.me/images/zfb.jpg', 'https://archive.diygod.me/images/wx.jpg'] diff --git a/.github/ISSUE_TEMPLATE/bug_report_en.yml b/.github/ISSUE_TEMPLATE/bug_report_en.yml index ac686b2dcc909a..de560dad5979cc 100644 --- a/.github/ISSUE_TEMPLATE/bug_report_en.yml +++ b/.github/ISSUE_TEMPLATE/bug_report_en.yml @@ -6,7 +6,7 @@ body: - type: markdown attributes: value: | - Please ensure you have read [documentation](https://docs.rsshub.app/en), and provide all the information required by this template, otherwise the issue will be closed immediately. + Please ensure you have read [documentation](https://docs.rsshub.app/), and provide all the information required by this template, otherwise the issue will be closed immediately. Due to the anti-crawling policy implemented by certain websites, some RSS routes provided by the demo will return status code 403. This is not an issue caused by RSSHub and please do not report it. - type: textarea diff --git a/.github/ISSUE_TEMPLATE/feature_request_en.yml b/.github/ISSUE_TEMPLATE/feature_request_en.yml index ed5db239e07fe0..7aee701cc8e5d9 100644 --- a/.github/ISSUE_TEMPLATE/feature_request_en.yml +++ b/.github/ISSUE_TEMPLATE/feature_request_en.yml @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Please ensure the feature requested is not listed in [documentation](https://docs.rsshub.app/en) or [issue](https://github.com/DIYgod/RSSHub/issues), and is not a [new RSS proposal](https://github.com/DIYgod/RSSHub/issues/new?assignees=&labels=RSS+proposal&template=rss_request_en.yml), and provide all the information required by this template. + Please ensure the feature requested is not listed in [documentation](https://docs.rsshub.app/) or [issue](https://github.com/DIYgod/RSSHub/issues), and is not a [new RSS proposal](https://github.com/DIYgod/RSSHub/issues/new?assignees=&labels=RSS+proposal&template=rss_request_en.yml), and provide all the information required by this template. Otherwise the issue will be closed immediately. - type: textarea diff --git a/.github/ISSUE_TEMPLATE/rss_request_en.yml b/.github/ISSUE_TEMPLATE/rss_request_en.yml index 85ad25a14bb3a6..0f1efc64b5c915 100644 --- a/.github/ISSUE_TEMPLATE/rss_request_en.yml +++ b/.github/ISSUE_TEMPLATE/rss_request_en.yml @@ -1,4 +1,4 @@ -name: 🍰 RSS Proposal +name: 🧡 RSS Proposal description: Submit a new RSS proposal labels: ['RSS proposal'] @@ -7,7 +7,7 @@ body: - type: markdown attributes: value: | - Please ensure the RSS proposal is not listed in [documentation](https://docs.rsshub.app/en) or [issue](https://github.com/DIYgod/RSSHub/issues), website doesn't provide this kind of RSS feed, and provide all the information required by this template. + Please ensure the RSS proposal is not listed in [documentation](https://docs.rsshub.app/) or [issue](https://github.com/DIYgod/RSSHub/issues), website doesn't provide this kind of RSS feed, and provide all the information required by this template. Otherwise the issue will be closed immediately. We are flooded with feature requests and short-handed, please try to make it yourself, the [guide](https://docs.rsshub.app/joinus) is a good place to start. Submit a pull request when done! diff --git a/.github/ISSUE_TEMPLATE/rss_request_zh.yml b/.github/ISSUE_TEMPLATE/rss_request_zh.yml index a737cdcbf0d3ff..2970aa6381e330 100644 --- a/.github/ISSUE_TEMPLATE/rss_request_zh.yml +++ b/.github/ISSUE_TEMPLATE/rss_request_zh.yml @@ -1,4 +1,4 @@ -name: 🍰 RSS 提案 +name: 🧡 RSS 提案 description: 提交新的 RSS 提案 labels: ['RSS proposal'] diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index fb401f50abd56d..e6ae2dd496121c 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,6 +1,6 @@ ## Involved Issue / 该 PR 相关 Issue @@ -15,7 +15,7 @@ Fail to comply will result in your pull request being closed automatically. 请在 `routes` 区域填写以 / 开头的完整路由地址，否则你的 PR 将会被无条件关闭。 如果路由包含在文档中列出可以完全穷举的参数（例如分类），请依次全部列出。 -```route +```routes /some/route /some/other/route /dont/use/this/or/modify/it @@ -32,14 +32,9 @@ If your changes are not related to route, please fill in `routes` section with ` ## New RSS Route Checklist / 新 RSS 路由检查表 - [ ] New Route / 新的路由 - - [ ] Follows [v2 Script Standard](https://docs.rsshub.app/joinus/advanced/script-standard) / 跟随 [v2 路由规范](https://docs.rsshub.app/zh/joinus/advanced/script-standard) -- [ ] Documentation / 文档说明 - - [ ] EN / 英文文档 - - [ ] CN / 中文文档 -- [ ] Full text / 全文获取 - - [ ] Use cache / 使用缓存 -- [ ] Anti-bot or rate limit / 反爬/频率限制 - - [ ] If yes, do your code reflect this sign? / 如果有, 是否有对应的措施? + - [ ] Follows [Script Standard](https://docs.rsshub.app/joinus/advanced/script-standard) / 跟随 [路由规范](https://docs.rsshub.app/zh/joinus/advanced/script-standard) +- [ ] Anti-bot or rate limit / 反爬/频率限制 + - [ ] If yes, do your code reflect this sign? / 如果有, 是否有对应的措施? - [ ] [Date and time](https://docs.rsshub.app/joinus/advanced/pub-date) / [日期和时间](https://docs.rsshub.app/zh/joinus/advanced/pub-date) - [ ] Parsed / 可以解析 - [ ] Correct time zone / 时区正确 diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4bfc2c1d8571e0..72d8543ed327e7 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,73 +4,19 @@ updates: directory: '/' schedule: interval: daily - time: '21:00' - open-pull-requests-limit: 10 + time: '08:00' + open-pull-requests-limit: 100 labels: - dependencies ignore: - # ESM only packages - - dependency-name: fanfou-sdk - versions: ['>=5.0.0'] - - dependency-name: got - versions: ['>=12.0.0'] - - dependency-name: ip-regex - versions: ['>=5.0.0'] - - dependency-name: query-string - versions: ['>=8.0.0'] - - dependency-name: rand-user-agent - versions: ['>=2.0.1'] - - dependency-name: remark - versions: ['>=14.0.0'] - - dependency-name: remark-frontmatter - versions: ['>=4.0.0'] - - dependency-name: remark-gfm - versions: ['>=2.0.0'] - - dependency-name: remark-parse - versions: ['>=10.0.0'] - - dependency-name: remark-preset-prettier - versions: ['>=1.0.0'] - - dependency-name: remark-stringify - versions: ['>=10.0.0'] - - dependency-name: string-width - versions: ['>=5.0.0'] - - dependency-name: unified - versions: ['>=10.0.0'] - - - package-ecosystem: npm - directory: '/website' - schedule: - interval: daily - time: '21:00' - open-pull-requests-limit: 10 - labels: - - dependencies - ignore: - # ESM only packages - - dependency-name: remark - versions: ['>=14.0.0'] - - dependency-name: remark-frontmatter - versions: ['>=4.0.0'] - - dependency-name: remark-gfm - versions: ['>=2.0.0'] - - dependency-name: remark-parse - versions: ['>=10.0.0'] - - dependency-name: remark-preset-prettier - versions: ['>=1.0.0'] - - dependency-name: remark-stringify - versions: ['>=10.0.0'] - - dependency-name: string-width - versions: ['>=5.0.0'] - groups: - docs: - patterns: - - '@docusaurus/*' + - dependency-name: jsrsasign + versions: ['>=11.0.0'] # no longer includes KJUR.crypto.Cipher for RSA - package-ecosystem: 'github-actions' directory: '/' schedule: interval: daily - time: '21:00' - open-pull-requests-limit: 10 + time: '08:00' + open-pull-requests-limit: 100 labels: - dependencies diff --git a/.github/labeler.yml b/.github/labeler.yml index d9a23aa3d088ee..5838333146c094 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -1,15 +1,17 @@ -'Route: v1': -- lib/router.js -- any: ['lib/routes/**/*.js', '!lib/routes/index.js'] +'Route: deprecated': +- changed-files: + - any-glob-to-any-file: ['lib/router.js'] + - all-globs-to-any-file: ['lib/routes-deprecated/**/*.js', '!lib/routes-deprecated/index.js'] -'Route: v2': -- 'lib/v2/**/*.js' +'Route': +- changed-files: + - any-glob-to-any-file: ['lib/routes/**/*.ts'] core enhancement: -- lib/routes/index.js -- any: ['lib/**', '!lib/radar-rules.js', '!lib/router.js', '!lib/routes/**', '!lib/v2/**'] +- changed-files: + - any-glob-to-any-file: ['lib/routes/index.ts'] + - all-globs-to-any-file: ['lib/**', '!lib/config.ts', '!lib/router.js', '!lib/routes/**', '!lib/routes-deprecated/**'] dependencies: -- package.json -- pnpm-lock.yaml -- yarn.lock +- changed-files: + - any-glob-to-any-file: ['package.json', 'pnpm-lock.yaml', 'yarn.lock'] diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml index d80d8bf661a8b3..e824ab20547146 100644 --- a/.github/workflows/build-assets.yml +++ b/.github/workflows/build-assets.yml @@ -1,38 +1,69 @@ -name: build assets +name: Build assets on: + workflow_dispatch: push: branches: - master paths: - - 'lib/**' - - 'scripts/workflow/*.js' + - 'lib/**/*.ts' jobs: build: runs-on: ubuntu-latest name: Build assets timeout-minutes: 5 + permissions: + contents: write steps: - name: Checkout uses: actions/checkout@v4 - name: Install pnpm - uses: pnpm/action-setup@v2 - with: - version: 8 + uses: pnpm/action-setup@v4 - name: Use Node.js Active LTS - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: lts/* cache: 'pnpm' - name: Install dependencies (yarn) run: pnpm i - name: Build assets - run: npm run build:all + run: pnpm build - name: Deploy - uses: peaceiris/actions-gh-pages@v3 + uses: peaceiris/actions-gh-pages@v4 with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./assets user_name: 'github-actions[bot]' user_email: '41898282+github-actions[bot]@users.noreply.github.com' + # prevent deleting build/test-full-routes.json which will break build:docs + keep_files: true + - name: Build docs + run: pnpm build:docs + - id: check-env + env: + DOCS_API_TOKEN: ${{ secrets.DOCS_API_TOKEN }} + if: ${{ env.DOCS_API_TOKEN != '' }} + run: echo "defined=true" >> $GITHUB_OUTPUT + - name: Checkout docs + uses: actions/checkout@v4 + if: steps.check-env.outputs.defined == 'true' + with: + repository: 'RSSNext/rsshub-docs' + token: ${{ secrets.DOCS_API_TOKEN }} + path: rsshub-docs + - name: Update docs + if: steps.check-env.outputs.defined == 'true' + run: | + cp -r ./assets/build/docs/en/* ./rsshub-docs/src/routes + cp -r ./assets/build/docs/zh/* ./rsshub-docs/src/zh/routes + cp ./lib/types.ts ./rsshub-docs/.vitepress/theme/types.ts + cp ./scripts/workflow/data.ts ./rsshub-docs/.vitepress/config/data.ts + - name: Commit docs + if: steps.check-env.outputs.defined == 'true' + run: | + cd rsshub-docs + git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" + git config --local user.name "github-actions[bot]" + git status + git diff-index --quiet HEAD || (git commit -m "chore: auto build https://github.com/$GITHUB_REPOSITORY/commit/$GITHUB_SHA" -a --no-verify && git push "https://${GITHUB_ACTOR}:${{ secrets.DOCS_API_TOKEN }}@github.com/RSSNext/rsshub-docs.git" HEAD:main) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 6ac7b5230ae772..7985f4e1501b46 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -13,28 +13,37 @@ name: 'CodeQL' on: push: - branches: [master] + branches: ['master'] pull_request: - # The branches below must be a subset of the branches above - branches: [master] + branches: ['master'] schedule: - cron: '15 9 * * 3' jobs: analyze: name: Analyze + # Runner size impacts CodeQL analysis time. To learn more, please see: + # - https://gh.io/recommended-hardware-resources-for-running-codeql + # - https://gh.io/supported-runners-and-hardware-resources + # - https://gh.io/using-larger-runners + # Consider using larger runners for possible analysis time improvements. runs-on: ubuntu-latest timeout-minutes: 10 permissions: + # required for all workflows + security-events: write + + # only required for workflows in private repositories actions: read contents: read - security-events: write strategy: fail-fast: false matrix: - language: ['javascript'] - # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] + language: ['javascript-typescript'] + # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ] + # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both + # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support steps: @@ -43,31 +52,32 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v2 + uses: github/codeql-action/init@v3 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. - # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs + # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). + # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild - uses: github/codeql-action/autobuild@v2 + uses: github/codeql-action/autobuild@v3 # ℹ️ Command-line programs to run using the OS shell. - # 📚 https://git.io/JvXDl + # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun - # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines - # and modify them (or add more) to build your code if your project - # uses a compiled language + # If the Autobuild fails above, remove it and uncomment the following three lines. + # modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance. # - run: | - # make bootstrap - # make release + # echo "Run, Build Application using script" + # ./location_of_script_within_repo/buildscript.sh - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v2 + uses: github/codeql-action/analyze@v3 + with: + category: '/language:${{matrix.language}}' diff --git a/.github/workflows/comment-on-issue.yml b/.github/workflows/comment-on-issue.yml index 61b518ba884bf2..f0fdb344784301 100644 --- a/.github/workflows/comment-on-issue.yml +++ b/.github/workflows/comment-on-issue.yml @@ -9,21 +9,22 @@ jobs: name: Call maintainers runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + issues: write + if: github.event.sender.login != 'issuehunt-oss[bot]' steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 # just need its cache + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: lts/* cache: 'pnpm' - - name: Install dependencies (pnpm) # needed since we need to parse markdown, so we also use got instead + - name: Install dependencies (pnpm) # import remark-parse and unified run: pnpm i - name: Generate feedback - uses: actions/github-script@v6 + uses: actions/github-script@v7 with: - github-token: ${{secrets.GITHUB_TOKEN}} + github-token: ${{ secrets.GITHUB_TOKEN }} script: | - const script = require(`${process.env.GITHUB_WORKSPACE}/scripts/workflow/test-issue/call-maintainer.js`) - return script({ github, context, core }) + const { default: callMaintainer } = await import('${{ github.workspace }}/scripts/workflow/test-issue/call-maintainer.mjs') + await callMaintainer({ github, context, core }) diff --git a/.github/workflows/dependabot-fork.yml b/.github/workflows/dependabot-fork.yml index b39ec2ae7cd20f..baeeff107b6ca8 100644 --- a/.github/workflows/dependabot-fork.yml +++ b/.github/workflows/dependabot-fork.yml @@ -13,7 +13,7 @@ jobs: uses: actions/checkout@v4 - name: Comment Dependabot PR - uses: thollander/actions-comment-pull-request@v2 + uses: thollander/actions-comment-pull-request@v3 with: message: '@dependabot ignore this dependency' GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml index 8e25b730e701c3..1d8c5b6bd546d6 100644 --- a/.github/workflows/docker-release.yml +++ b/.github/workflows/docker-release.yml @@ -1,4 +1,4 @@ -name: '[docker] CI for releases' +name: 'Docker Release' on: push: @@ -7,13 +7,9 @@ on: paths: - '.github/workflows/docker-release.yml' - 'lib/**' - - '!**/maintainer.js' - - '!**/radar.js' - - '!**/radar-rules.js' - - '!lib/v2/test/**' - - '!test/**' + - '!lib/**/*.test.ts' - 'Dockerfile' - workflow_dispatch: ~ + workflow_dispatch: {} jobs: check-env: @@ -33,10 +29,9 @@ jobs: runs-on: ubuntu-latest needs: check-env if: needs.check-env.outputs.check-docker == 'true' - timeout-minutes: 120 + timeout-minutes: 60 permissions: packages: write - contents: read id-token: write steps: - name: Checkout @@ -75,11 +70,12 @@ jobs: tags: | type=raw,value=latest,enable=true type=raw,value={{date 'YYYY-MM-DD'}},enable=true + type=sha,format=long,prefix=,enable=true flavor: latest=false - name: Build and push Docker image (ordinary version) id: build-and-push - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . push: true @@ -105,11 +101,12 @@ jobs: tags: | type=raw,value=chromium-bundled,enable=true type=raw,value=chromium-bundled-{{date 'YYYY-MM-DD'}},enable=true + type=sha,format=long,prefix=chromium-bundled-,enable=true flavor: latest=false - name: Build and push Docker image (Chromium-bundled version) id: build-and-push-chromium - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . build-args: PUPPETEER_SKIP_DOWNLOAD=0 @@ -136,7 +133,7 @@ jobs: - uses: actions/checkout@v4 - name: Docker Hub Description - uses: peter-evans/dockerhub-description@v3 + uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} diff --git a/.github/workflows/docker-test-cont.yml b/.github/workflows/docker-test-cont.yml new file mode 100644 index 00000000000000..be3c207be45aa4 --- /dev/null +++ b/.github/workflows/docker-test-cont.yml @@ -0,0 +1,119 @@ +name: PR - route test + +on: + workflow_run: + workflows: [PR - Docker build test] # open, reopen, synchronized, edited included + types: [completed] + +jobs: + testRoute: + name: Route test + runs-on: ubuntu-latest + permissions: + pull-requests: write + if: ${{ github.event.workflow_run.conclusion == 'success' }} # skip if unsuccessful + steps: + - uses: actions/checkout@v4 + + # https://github.com/orgs/community/discussions/25220 + - name: Search the PR that triggered this workflow + uses: potiuk/get-workflow-origin@v1_5 + id: source-run-info + with: + token: ${{ secrets.GITHUB_TOKEN }} + sourceRunId: ${{ github.event.workflow_run.id }} + + - name: Fetch PR data via GitHub API + uses: octokit/request-action@v2.x + id: pr-data + with: + route: GET /repos/{repo}/pulls/{number} + repo: ${{ github.repository }} + number: ${{ steps.source-run-info.outputs.pullRequestNumber }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Fetch affected routes + id: fetch-route + uses: actions/github-script@v7 + env: + PULL_REQUEST: ${{ steps.pr-data.outputs.data }} + with: + # by default, JSON format returned + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const PR = JSON.parse(process.env.PULL_REQUEST) + const body = PR.body + const number = PR.number + const sender = PR.user.login + const { default: identify } = await import('${{ github.workspace }}/scripts/workflow/test-route/identify.mjs') + return identify({ github, context, core }, body, number, sender) + + - name: Fetch Docker image + if: (env.TEST_CONTINUE) + uses: dawidd6/action-download-artifact@v7 + with: + workflow: ${{ github.event.workflow_run.workflow_id }} + run_id: ${{ github.event.workflow_run.id }} + name: docker-image + path: ../artifacts-${{ github.run_id }} + + - name: Import Docker image and set up Docker container + if: (env.TEST_CONTINUE) + working-directory: ../artifacts-${{ github.run_id }} + run: | + set -ex + zstd -d --stdout rsshub.tar.zst | docker load + docker run -d \ + --name rsshub \ + -e NODE_ENV=dev \ + -e LOGGER_LEVEL=debug \ + -e ALLOW_USER_HOTLINK_TEMPLATE=true \ + -e ALLOW_USER_SUPPLY_UNSAFE_DOMAIN=true \ + -p 1200:1200 \ + rsshub:latest + + - uses: pnpm/action-setup@v4 + + - uses: actions/setup-node@v4 + if: (env.TEST_CONTINUE) + with: + node-version: lts/* + cache: 'pnpm' + + - name: Install dependencies (pnpm) # require js-beautify + if: (env.TEST_CONTINUE) + run: pnpm i + + - name: Generate feedback + if: (env.TEST_CONTINUE) + id: generate-feedback + timeout-minutes: 10 + uses: actions/github-script@v7 + env: + TEST_BASEURL: http://localhost:1200 + TEST_ROUTES: ${{ steps.fetch-route.outputs.result }} + PULL_REQUEST: ${{ steps.pr-data.outputs.data }} + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const PR = JSON.parse(process.env.PULL_REQUEST) + const link = process.env.TEST_BASEURL + const routes = JSON.parse(process.env.TEST_ROUTES) + const number = PR.number + core.info(`${link}, ${routes}, ${number}`) + const { default: test } = await import('${{ github.workspace }}/scripts/workflow/test-route/test.mjs') + await test({ github, context, core }, link, routes, number) + + - name: Pull Request Labeler + if: ${{ failure() }} + uses: actions-cool/issues-helper@v3 + with: + actions: 'add-labels' + token: ${{ secrets.GITHUB_TOKEN }} + issue-number: ${{ steps.source-run-info.outputs.pullRequestNumber }} + labels: 'Auto: Route Test Failed' + + - name: Print Docker container logs + if: (env.TEST_CONTINUE) + run: docker logs rsshub # logs/combined.log? Not so readable... diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 7571ecb60f6007..79144429814db9 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -1,5 +1,3 @@ -# name: '[docker] CI for build tests' -# https://github.community/t/215358 name: PR - Docker build test on: @@ -9,19 +7,12 @@ on: paths: - '.github/workflows/docker-test.yml' - 'lib/**' - - '!**/maintainer.js' - - '!**/radar.js' - - '!**/radar-rules.js' - 'Dockerfile' - 'package.json' - 'pnpm-lock.yaml' types: [opened, reopened, synchronize, edited] # Please, always create a pull request instead of push to master. -permissions: - contents: read - pull-requests: write - concurrency: group: docker-test-${{ github.ref_name }} cancel-in-progress: true @@ -29,8 +20,11 @@ concurrency: jobs: test: name: Docker build & tests + permissions: + pull-requests: write + attestations: write runs-on: ubuntu-latest - timeout-minutes: 15 + timeout-minutes: 10 steps: - name: Checkout uses: actions/checkout@v4 @@ -46,7 +40,7 @@ jobs: flavor: latest=true - name: Build Docker image - uses: docker/build-push-action@v5 + uses: docker/build-push-action@v6 with: context: . build-args: PUPPETEER_SKIP_DOWNLOAD=0 # also test bundling Chromium @@ -65,17 +59,17 @@ jobs: actions: 'add-labels' token: ${{ secrets.GITHUB_TOKEN }} issue-number: ${{ github.event.pull_request.number }} - labels: 'Route Test: Failed' + labels: 'Auto: Route Test Failed' - name: Test Docker image run: bash scripts/docker/test-docker.sh - name: Export Docker image - run: docker save rsshub:latest | gzip -1cf > rsshub.tar.gz + run: docker save rsshub:latest | zstdmt -o rsshub.tar.zst - name: Upload Docker image - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: docker-image - path: rsshub.tar.gz + path: rsshub.tar.zst retention-days: 1 diff --git a/.github/workflows/docs-search-index.yml b/.github/workflows/docs-search-index.yml deleted file mode 100644 index 2cca71ac98c38f..00000000000000 --- a/.github/workflows/docs-search-index.yml +++ /dev/null @@ -1,58 +0,0 @@ -name: Update meilisearch index - -on: - push: - branches: - - master - paths: - - '.github/workflows/docs-search-index.yml' - - 'scripts/docs-scraper/docs.rsshub.app.json' - - 'website/**' - workflow_dispatch: ~ - schedule: - - cron: '44 1 * * 1' - -concurrency: - group: docs-search-index - -jobs: - scrape-docs: - runs-on: ubuntu-latest - timeout-minutes: 15 - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Pull image - run: docker pull getmeili/docs-scraper - - name: Wait for Netlify to finish - run: sleep 2m - if: github.event_name == 'push' - - name: Run docs-scraper - env: - HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} - API_KEY: ${{ secrets.MEILISEARCH_API_KEY }} - CONFIG_FILE_PATH: ${{ github.workspace }}/scripts/docs-scraper/docs.rsshub.app.json - run: | - docker run -t --rm \ - -e MEILISEARCH_HOST_URL=$HOST_URL \ - -e MEILISEARCH_API_KEY=$API_KEY \ - -v $CONFIG_FILE_PATH:/docs-scraper/config.json \ - getmeili/docs-scraper pipenv run ./docs_scraper config.json - - name: Swap index - env: - HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} - API_KEY: ${{ secrets.MEILISEARCH_API_KEY }} - run: | - curl \ - -X POST $HOST_URL/swap-indexes \ - -H "Authorization: Bearer $API_KEY" \ - -H 'Content-Type: application/json' \ - -d '[{"indexes":["rsshub","rsshub-tmp"]}]' - - name: Delete old index - env: - HOST_URL: ${{ secrets.MEILISEARCH_HOST_URL }} - API_KEY: ${{ secrets.MEILISEARCH_API_KEY }} - run: | - curl \ - -X DELETE $HOST_URL/indexes/rsshub-tmp \ - -H "Authorization: Bearer $API_KEY" diff --git a/.github/workflows/format.yml b/.github/workflows/format.yml index e82387e8486fb9..a2ca9899fa5b32 100644 --- a/.github/workflows/format.yml +++ b/.github/workflows/format.yml @@ -1,13 +1,10 @@ -name: format +name: Format on: push: branches: - master -permissions: - contents: read - jobs: format: permissions: @@ -18,16 +15,12 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: lts/* cache: 'pnpm' - run: pnpm i - - run: pnpm i - working-directory: website - run: npm run format - name: Commit files run: | diff --git a/.github/workflows/issue-command.yml b/.github/workflows/issue-command.yml index 1206b0c514e451..b80c2b5ea2dfa4 100644 --- a/.github/workflows/issue-command.yml +++ b/.github/workflows/issue-command.yml @@ -4,15 +4,15 @@ on: issue_comment: types: [created] -permissions: - contents: read - jobs: rebase: name: Automatic Rebase if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/rebase') && github.event.comment.author_association == 'COLLABORATOR' runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: write + pull-requests: write steps: - name: Checkout the latest code uses: actions/checkout@v4 @@ -21,7 +21,7 @@ jobs: - name: Automatic Rebase uses: cirrus-actions/rebase@1.8 env: - GITHUB_TOKEN: ${{ secrets.TOKEN_SUPER }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} self-assign: name: Self Assign @@ -29,10 +29,9 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: - contents: read issues: write steps: - - uses: bdougie/take-action@v1.5 + - uses: bdougie/take-action@v1.6.1 with: token: ${{ secrets.GITHUB_TOKEN }} trigger: '/wip' @@ -43,19 +42,36 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 5 permissions: - contents: read + attestations: write issues: write + pull-requests: write steps: + - name: Fetch PR data (for PR) + if: github.event.issue.pull_request + uses: octokit/request-action@v2.x + id: pr-data + with: + route: GET /repos/{repo}/pulls/{number} + repo: ${{ github.repository }} + number: ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout + if: ${{ !github.event.issue.pull_request }} uses: actions/checkout@v4 - - name: Install pnpm - uses: pnpm/action-setup@v2 + - name: Checkout PR + if: github.event.issue.pull_request + uses: actions/checkout@v4 with: - version: 8 + ref: ${{ fromJson(steps.pr-data.outputs.data).head.ref }} + + - name: Install pnpm + uses: pnpm/action-setup@v4 - name: Use Node.js Active LTS - uses: actions/setup-node@v3 + uses: actions/setup-node@v4 with: node-version: lts/* cache: 'pnpm' @@ -64,8 +80,8 @@ jobs: run: pnpm i && pnpm rb - name: Fetch affected routes - id: fetchRoute - uses: actions/github-script@v6 + id: fetch-route + uses: actions/github-script@v7 env: EVENT: ${{ toJson(github.event) }} with: @@ -74,8 +90,12 @@ jobs: const body = event.comment.body const number = event.issue.number const sender = event.comment.user.login - const script = require(`${process.env.GITHUB_WORKSPACE}/scripts/workflow/test-route/identify.js`) - return script({ github, context, core }, body, number, sender) + const { default: identify } = await import('${{ github.workspace }}/scripts/workflow/test-route/identify.mjs') + return identify({ github, context, core }, body, number, sender) + + - name: Build RSSHub + if: env.TEST_CONTINUE + run: pnpm build - name: Start RSSHub if: env.TEST_CONTINUE @@ -88,10 +108,10 @@ jobs: - name: Generate feedback if: env.TEST_CONTINUE - uses: actions/github-script@v6 + uses: actions/github-script@v7 env: TEST_BASEURL: http://localhost:1200 - TEST_ROUTES: ${{ steps.fetchRoute.outputs.result }} + TEST_ROUTES: ${{ steps.fetch-route.outputs.result }} EVENT: ${{ toJson(github.event) }} with: script: | @@ -100,16 +120,15 @@ jobs: const routes = JSON.parse(process.env.TEST_ROUTES) const number = event.issue.number core.info(`${link}, ${routes}, ${number}`) - const got = require("got") - const script = require(`${process.env.GITHUB_WORKSPACE}/scripts/workflow/test-route/test.js`) - return script({ github, context, core, got }, link, routes, number) + const { default: test } = await import('${{ github.workspace }}/scripts/workflow/test-route/test.mjs') + await test({ github, context, core }, link, routes, number) - name: Print logs - if: (env.TEST_CONTINUE) + if: env.TEST_CONTINUE run: cat ${{ github.workspace }}/logs/combined.log - name: Upload Artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs path: logs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000000..4f317a38646bd1 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,83 @@ +name: Linter + +# https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request +# pull_request includes [opened, reopened, synchronize] events by default +# 'edited' is required for title-lint +on: + push: {} + pull_request: + types: [opened, reopened, synchronize, edited] + pull_request_target: + types: [opened, reopened, synchronize, edited] + +jobs: + # eslint: + # name: ESLint + # if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + # runs-on: ubuntu-latest + # timeout-minutes: 5 + # steps: + # - uses: actions/checkout@v4 + # - uses: pnpm/action-setup@v4 + # with: + # version: 9 + # - uses: actions/setup-node@v4 + # with: + # node-version: lts/* + # cache: 'pnpm' + # - run: pnpm i + # - name: Lint + # run: pnpm run lint + +# https://github.com/actions/starter-workflows/blob/main/code-scanning/eslint.yml + eslint-warning: + name: Lint + if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + security-events: write + steps: + - uses: actions/checkout@v4 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 + with: + node-version: lts/* + cache: 'pnpm' + - run: pnpm i + - name: Lint + run: pnpm run lint + --format @microsoft/eslint-formatter-sarif + --output-file eslint-results.sarif + continue-on-error: true + - name: Upload analysis results to GitHub + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: eslint-results.sarif + wait-for-processing: true + +# https://github.com/amannn/action-semantic-pull-request + title-lint: + if: ${{ github.event_name == 'pull_request_target' && github.repository == 'DIYgod/RSSHub' }} + name: Validate PR title + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: amannn/action-semantic-pull-request@v5 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ignoreLabels: | + dependencies + wip: true + + labeler: + name: Pull Request Labeler + if: ${{ github.event_name == 'pull_request_target' && github.actor != 'dependabot[bot]' && github.repository == 'DIYgod/RSSHub' }} + permissions: + pull-requests: write + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/labeler@v5 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/npm-publish.yml b/.github/workflows/npm-publish.yml index 03e77142c59db5..bd6666ae3af8f8 100644 --- a/.github/workflows/npm-publish.yml +++ b/.github/workflows/npm-publish.yml @@ -1,4 +1,4 @@ -name: publish +name: npm Publish on: push: @@ -7,12 +7,8 @@ on: paths: - '.github/workflows/npm-publish.yml' - 'lib/**' - - '!**/maintainer.js' - - '!**/radar.js' - - '!**/radar-rules.js' permissions: - contents: read id-token: write jobs: @@ -25,10 +21,8 @@ jobs: HUSKY: 0 steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: lts/* cache: 'pnpm' diff --git a/.github/workflows/pr-deploy-route-test.yml b/.github/workflows/pr-deploy-route-test.yml deleted file mode 100644 index 900bb493206c48..00000000000000 --- a/.github/workflows/pr-deploy-route-test.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: PR - route test -on: - workflow_run: - workflows: [ PR - Docker build test ] # open, reopen, synchronized, edited included - types: [ completed ] - -jobs: - testRoute: - name: Route test - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} # skip if unsuccessful - timeout-minutes: 10 - steps: - - uses: actions/checkout@v4 - - # https://github.com/orgs/community/discussions/25220 - - name: Search the PR that triggered this workflow - uses: potiuk/get-workflow-origin@v1_5 - id: source-run-info - with: - token: ${{ secrets.GITHUB_TOKEN }} - sourceRunId: ${{ github.event.workflow_run.id }} - - - name: Fetch PR data via GitHub API - uses: octokit/request-action@v2.x - id: pr-data - with: - route: GET /repos/{repo}/pulls/{number} - repo: ${{ github.repository }} - number: ${{ steps.source-run-info.outputs.pullRequestNumber }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - - name: Fetch affected routes - id: fetchRoute - uses: actions/github-script@v6 - env: - PULL_REQUEST: ${{ steps.pr-data.outputs.data }} - with: - # by default, JSON format returned - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const PR = JSON.parse(process.env.PULL_REQUEST) - const body = PR.body - const number = PR.number - const sender = PR.user.login - const script = require(`${process.env.GITHUB_WORKSPACE}/scripts/workflow/test-route/identify.js`) - return script({ github, context, core }, body, number, sender) - - - name: Fetch Docker image - if: (env.TEST_CONTINUE) - uses: dawidd6/action-download-artifact@v2 - with: - workflow: ${{ github.event.workflow_run.workflow_id }} - run_id: ${{ github.event.workflow_run.id }} - - - name: Import Docker image and set up Docker container - if: (env.TEST_CONTINUE) - run: | - set -ex - gzip -cvd docker-image/rsshub.tar.gz | docker load - docker run -d \ - --name rsshub \ - -e NODE_ENV=dev \ - -e LOGGER_LEVEL=debug \ - -e ALLOW_USER_HOTLINK_TEMPLATE=true \ - -e ALLOW_USER_SUPPLY_UNSAFE_DOMAIN=true \ - -p 1200:1200 \ - rsshub:latest - - - uses: pnpm/action-setup@v2 - with: - version: 8 - - - uses: actions/setup-node@v3 # just need its cache - if: (env.TEST_CONTINUE) - with: - node-version: lts/* - cache: 'pnpm' - - - name: Install dependencies (pnpm) # `got` needed since `github.request` disallows HTTP requests - if: (env.TEST_CONTINUE) - run: pnpm i - - - name: Generate feedback - if: (env.TEST_CONTINUE) - uses: actions/github-script@v6 - env: - TEST_BASEURL: http://localhost:1200 - TEST_ROUTES: ${{ steps.fetchRoute.outputs.result }} - PULL_REQUEST: ${{ steps.pr-data.outputs.data }} - with: - github-token: ${{secrets.GITHUB_TOKEN}} - script: | - const PR = JSON.parse(process.env.PULL_REQUEST) - const link = process.env.TEST_BASEURL - const routes = JSON.parse(process.env.TEST_ROUTES) - const number = PR.number - core.info(`${link}, ${routes}, ${number}`) - const got = require("got"); - const script = require(`${process.env.GITHUB_WORKSPACE}/scripts/workflow/test-route/test.js`) - return script({ github, context, core, got }, link, routes, number) - - - name: Print Docker container logs - if: (env.TEST_CONTINUE) - run: docker logs rsshub # logs/combined.log? Not so readable... diff --git a/.github/workflows/pr-lint.yml b/.github/workflows/pr-lint.yml deleted file mode 100644 index 3a1a784a5900f0..00000000000000 --- a/.github/workflows/pr-lint.yml +++ /dev/null @@ -1,95 +0,0 @@ -name: Linter - -on: [push, pull_request, pull_request_target] - -jobs: - eslint: - name: ESLint - if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - cache: 'pnpm' - - run: pnpm i - - name: Lint - run: pnpm run lint - - eslint-warning: - name: Lint - if: ${{ github.event_name == 'push' || github.event_name == 'pull_request' }} - runs-on: ubuntu-latest - timeout-minutes: 5 - permissions: - contents: read - security-events: write - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 - with: - node-version: lts/* - cache: 'pnpm' - - run: pnpm i - - name: Lint - run: pnpm run lint - --format @microsoft/eslint-formatter-sarif - --output-file eslint-results.sarif - continue-on-error: true - - name: Upload analysis results to GitHub - uses: github/codeql-action/upload-sarif@v2 - with: - sarif_file: eslint-results.sarif - wait-for-processing: true - - titleLint: - if: ${{ github.event_name == 'pull_request_target' && github.repository == 'DIYgod/RSSHub' }} - name: Validate PR title - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: amannn/action-semantic-pull-request@v5 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - ignoreLabels: | - dependencies - wip: true - - labeler: - name: Pull Request Labeler - if: ${{ github.event_name == 'pull_request_target' && github.actor != 'dependabot[bot]' }} - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: actions/labeler@v4 - with: - repo-token: ${{ secrets.GITHUB_TOKEN }} - - chatgpt-review: - name: ChatGPT PR reviewer - if: ${{ github.event_name == 'pull_request_target' && false }} - permissions: - contents: read - pull-requests: write - runs-on: ubuntu-latest - timeout-minutes: 5 - steps: - - uses: fluxninja/openai-pr-reviewer@v1 - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - with: - openai_base_url: ${{ secrets.OPENAI_API_ENDPOINT }} - openai_model_temperature: '1.0' - disable_release_notes: true diff --git a/.github/workflows/semgrep.yml b/.github/workflows/semgrep.yml index d67cc7ddccd77a..5fb469de005d95 100644 --- a/.github/workflows/semgrep.yml +++ b/.github/workflows/semgrep.yml @@ -1,26 +1,33 @@ +name: Semgrep + +# https://semgrep.dev/docs/semgrep-ci/sample-ci-configs/#sample-github-actions-configuration-file on: pull_request_target: branches: - - master - paths: - - .github/workflows/semgrep.yml + - master push: branches: - - master - paths: - - .github/workflows/semgrep.yml + - master schedule: - # random HH:MM to avoid a load spike on GitHub Actions at 00:00 - - cron: 21 20 * * * -name: Semgrep + # random HH:MM to avoid a load spike on GitHub Actions at 00:00 + - cron: 21 20 * * * + jobs: semgrep: name: Scan runs-on: ubuntu-latest - env: - SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} container: image: returntocorp/semgrep + if: (github.triggering_actor != 'dependabot[bot]') + permissions: + security-events: write steps: - - uses: actions/checkout@v4 - - run: semgrep ci + - uses: actions/checkout@v4 + - run: semgrep ci --sarif > semgrep.sarif + env: + SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + - name: Upload SARIF file for GitHub Advanced Security Dashboard + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: semgrep.sarif + if: always() diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 90701a832db464..c11a215094233a 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,5 @@ name: 'Close stale issues and PRs' + on: schedule: - cron: '31 23 * * *' @@ -11,7 +12,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: # Don't stale issues days-before-issue-stale: -1 @@ -21,3 +22,6 @@ jobs: This PR is stale because it has been opened for more than 3 weeks with no activity. Comment or this will be closed in 7 days. close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.' close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.' + exempt-issue-labels: 'dependencies,wait for upstream' + exempt-pr-labels: 'dependencies,wait for upstream' + any-of-issue-labels: 'more data required' diff --git a/.github/workflows/test-full-routes.yml b/.github/workflows/test-full-routes.yml new file mode 100644 index 00000000000000..ce46651922ba53 --- /dev/null +++ b/.github/workflows/test-full-routes.yml @@ -0,0 +1,39 @@ +name: Build assets (Full Routes Test Result) + +on: + workflow_dispatch: + schedule: + - cron: '0 0 * * *' + +jobs: + build: + runs-on: ubuntu-latest + name: Build assets + timeout-minutes: 120 + permissions: + contents: write + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Install pnpm + uses: pnpm/action-setup@v4 + - name: Use Node.js Active LTS + uses: actions/setup-node@v4 + with: + node-version: lts/* + cache: 'pnpm' + - name: Install dependencies (yarn) + run: pnpm i + - name: Build assets + run: pnpm build + - name: Build full routes test result + continue-on-error: true + run: pnpm vitest:fullroutes + - name: Deploy + uses: peaceiris/actions-gh-pages@v4 + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: ./assets + user_name: 'github-actions[bot]' + user_email: '41898282+github-actions[bot]@users.noreply.github.com' + keep_files: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63356b25073d1a..f335abdebb7725 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,25 +1,21 @@ -name: test +name: Test on: push: branches-ignore: - 'dependabot/**' paths: - - 'test/**' - 'lib/**' - - '!**/maintainer.js' - - '!**/radar.js' - - '!**/radar-rules.js' - 'package.json' - 'pnpm-lock.yaml' - '.github/workflows/test.yml' - pull_request: ~ + pull_request: {} permissions: - contents: read + checks: write jobs: - jest: + vitest: runs-on: ubuntu-latest timeout-minutes: 10 services: @@ -31,14 +27,12 @@ jobs: strategy: fail-fast: false matrix: - node-version: [ 18, 20 ] - name: Jest on Node ${{ matrix.node-version }} + node-version: [ latest, lts/*, lts/-1 ] + name: Vitest on Node ${{ matrix.node-version }} steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} cache: 'pnpm' @@ -46,13 +40,15 @@ jobs: run: pnpm i - name: Run postinstall script for dependencies run: pnpm rb + - name: Build routes + run: pnpm build - name: Test all and generate coverage - run: pnpm run jest:coverage + run: pnpm run vitest:coverage --reporter=github-actions env: REDIS_URL: redis://localhost:${{ job.services.redis.ports[6379] }}/ - name: Upload coverage to Codecov - if: ${{ matrix.node-version == '18' }} - uses: codecov/codecov-action@v3 + if: ${{ matrix.node-version == 'lts/*' }} + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} # not required for public repos as documented, but seems broken @@ -62,39 +58,42 @@ jobs: strategy: fail-fast: false matrix: - node-version: [ 18, 20 ] + node-version: [ latest, lts/*, lts/-1 ] chromium: - name: bundled Chromium dependency: '' - environment: '{}' + environment: '{ "PUPPETEER_SKIP_DOWNLOAD": "0" }' - name: Chromium from Ubuntu dependency: chromium-browser - environment: '{ "CHROMIUM_EXECUTABLE_PATH": "chromium-browser" }' + environment: '{ "PUPPETEER_SKIP_DOWNLOAD": "1" }' - name: Chrome from Google dependency: google-chrome-stable - environment: '{ "CHROMIUM_EXECUTABLE_PATH": "google-chrome-stable" }' - name: Jest puppeteer on Node ${{ matrix.node-version }} with ${{ matrix.chromium.name }} + environment: '{ "PUPPETEER_SKIP_DOWNLOAD": "1" }' + name: Vitest puppeteer on Node ${{ matrix.node-version }} with ${{ matrix.chromium.name }} steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} cache: 'pnpm' - name: Install dependencies (pnpm) run: pnpm i + env: ${{ fromJSON(matrix.chromium.environment) }} - name: Run postinstall script for dependencies run: pnpm rb + env: ${{ fromJSON(matrix.chromium.environment) }} + - name: Build routes + run: pnpm build + env: ${{ fromJSON(matrix.chromium.environment) }} - name: Install Chromium if: ${{ matrix.chromium.dependency != '' }} # 'chromium-browser' from Ubuntu APT repo is a dummy package. Its version (85.0.4183.83) means # nothing since it calls Snap (disgusting!) to install Chromium, which should be up-to-date. - # That's not really a problem since the Chromium-bundled Docker image is based on Debian bullseye, + # That's not really a problem since the Chromium-bundled Docker image is based on Debian bookworm, # which provides up-to-date native packages. run: | - set -ex + set -eux curl -s "https://dl.google.com/linux/linux_signing_key.pub" | gpg --dearmor | sudo tee /etc/apt/trusted.gpg.d/google-chrome.gpg > /dev/null echo "deb [arch=amd64] https://dl.google.com/linux/chrome/deb/ stable main" | @@ -102,59 +101,41 @@ jobs: sudo apt-get update sudo apt-get install -yq --no-install-recommends ${{ matrix.chromium.dependency }} - name: Test puppeteer - run: pnpm run jest puppeteer + run: | + set -eux + export CHROMIUM_EXECUTABLE_PATH="$(which ${{ matrix.chromium.dependency }})" + pnpm run vitest puppeteer env: ${{ fromJSON(matrix.chromium.environment) }} - docs: - runs-on: ubuntu-latest - timeout-minutes: 10 - strategy: - fail-fast: false - matrix: - node-version: [ 18, 20 ] - defaults: - run: - working-directory: website - name: Build docs on Node ${{ matrix.node-version }} - steps: - - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 - with: - node-version: ${{ matrix.node-version }} - cache: 'pnpm' - cache-dependency-path: website/pnpm-lock.yaml - - run: pnpm i - - name: Build docs - run: pnpm run build - working-directory: website - all: runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + attestations: write strategy: fail-fast: false matrix: - node-version: [ 18, 20 ] + node-version: [ 23, 22, 20 ] name: Build radar and maintainer on Node ${{ matrix.node-version }} steps: - uses: actions/checkout@v4 - - uses: pnpm/action-setup@v2 - with: - version: 8 - - uses: actions/setup-node@v3 + - uses: pnpm/action-setup@v4 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} cache: 'pnpm' - run: pnpm i - name: Build radar and maintainer - run: npm run build:all + run: npm run build + - name: Upload assets + uses: actions/upload-artifact@v4 + with: + name: generated-assets-${{ matrix.node-version }} + path: assets/build/ automerge: - if: github.actor == 'dependabot[bot]' && github.event_name == 'pull_request' - needs: [ jest, puppeteer, docs, all ] + if: github.triggering_actor == 'dependabot[bot]' && github.event_name == 'pull_request' + needs: [ vitest, puppeteer, all ] runs-on: ubuntu-latest permissions: pull-requests: write diff --git a/.github/workflows/yarn-lock-changes.yml b/.github/workflows/yarn-lock-changes.yml deleted file mode 100644 index 00b14898fe60af..00000000000000 --- a/.github/workflows/yarn-lock-changes.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Yarn Lock Changes -on: - pull_request: - paths: - - '.github/workflows/yarn-lock-changes.yml' - - 'yarn.lock' - - 'docs/yarn.lock' - -jobs: - yarn_lock_changes: - runs-on: ubuntu-latest - timeout-minutes: 5 - # Permission overwrite is required for Dependabot PRs, see https://github.com/marketplace/actions/yarn-lock-changes#-common-issues. - permissions: - pull-requests: write - steps: - - name: Checkout - uses: actions/checkout@v4 - - name: Yarn Lock Changes - uses: Simek/yarn-lock-changes@v0.11.2 - with: - token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 0f1bcc4871414c..368a4c547ba8e7 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ .DS_Store -.env +.env* .eslintcache .idea .log @@ -19,6 +19,7 @@ coverage docs/.vuepress/dist node_modules tmp +dist Session.vim combined.log diff --git a/.gitpod.yml b/.gitpod.yml index 1b49b8f8bd39b6..0ee2c93f22e2c0 100644 --- a/.gitpod.yml +++ b/.gitpod.yml @@ -14,7 +14,7 @@ tasks: sudo apt update sudo apt install -y ca-certificates fonts-liberation libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 libdbus-1-3 libexpat1 libgbm1 libglib2.0-0 libnspr4 libnss3 libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 libxfixes3 libxkbcommon0 libxrandr2 wget xdg-util sudo apt install -y redis-server - init: pnpm i && pnpm i -C website && pnpm rb + init: pnpm i && pnpm rb - name: app command: pnpm run dev openMode: tab-after @@ -32,15 +32,7 @@ vscode: - EditorConfig.EditorConfig - esbenp.prettier-vscode - deepscan.vscode-deepscan - - rangav.vscode-thunder-client - sonarsource.sonarlint-vscode + # - VASubasRaj.flashpost not available on Open VSX, Thunder Client is paywalled in WSL/Codespaces/SSH > 2.30.0 + - unifiedjs.vscode-mdx # - ZihanLi.at-helper not available on Open VSX - -github: - prebuilds: - master: true - branches: true - pullRequests: false - pullRequestsFromForks: false - addCheck: false - addComment: false diff --git a/.husky/pre-commit b/.husky/pre-commit old mode 100755 new mode 100644 index d24fdfc601b9ff..c27d8893a99490 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1 @@ -#!/usr/bin/env sh -. "$(dirname -- "$0")/_/husky.sh" - -npx lint-staged +lint-staged diff --git a/.markdownlint.jsonc b/.markdownlint.jsonc deleted file mode 100644 index e426ea78adf606..00000000000000 --- a/.markdownlint.jsonc +++ /dev/null @@ -1,11 +0,0 @@ -{ - "MD007": { "indent": 4 }, // ul-indent - Unordered list indentation - "MD013": false, // line-length - Line length - "MD014": false, // commands-show-output - Dollar signs used before commands without showing output - "MD024": { "siblings_only": true }, // no-duplicate-heading/no-duplicate-header - Multiple headings with the same content - "MD030": { "ul_single": 3, "ol_single": 2, "ul_multi": 3, "ol_multi": 2 }, // list-marker-space - Spaces after list markers - "MD033": false, // no-inline-html - Inline HTML - "MD036": false, // no-emphasis-as-heading/no-emphasis-as-header - Emphasis used instead of a heading - "MD040": false, // fenced-code-language - Fenced code blocks should have a language specified - "MD051": false // link-fragments - Link fragments should be valid -} diff --git a/.npmrc b/.npmrc new file mode 100644 index 00000000000000..74538ab74e5dd9 --- /dev/null +++ b/.npmrc @@ -0,0 +1,2 @@ +package-lock=true +package-manager-strict=false diff --git a/.nvmrc b/.nvmrc deleted file mode 100644 index a77793ecc5200b..00000000000000 --- a/.nvmrc +++ /dev/null @@ -1 +0,0 @@ -lts/hydrogen diff --git a/.prettierignore b/.prettierignore index 8ee16a7bc609f1..b6b684c5ac07b0 100644 --- a/.prettierignore +++ b/.prettierignore @@ -1,8 +1,4 @@ -package.json -docs/.vuepress/dist -package-lock.json -.github/*.md -renovate.json -coverage -.vscode/ -website +lib/routes-deprecated +lib/router.js +babel.config.js +scripts/docker/minify-docker.js diff --git a/.puppeteerrc.cjs b/.puppeteerrc.cjs new file mode 100644 index 00000000000000..a4e6d37234ef19 --- /dev/null +++ b/.puppeteerrc.cjs @@ -0,0 +1,9 @@ +const path = require('path'); + +/** + * @type {import("puppeteer").Configuration} + */ +module.exports = { + // Changes the cache location for Puppeteer. + cacheDirectory: path.join(__dirname, 'node_modules', '.cache', 'puppeteer'), +}; diff --git a/.puppeteerrc.js b/.puppeteerrc.js deleted file mode 100644 index 06843b6b1575cc..00000000000000 --- a/.puppeteerrc.js +++ /dev/null @@ -1,9 +0,0 @@ -const { join } = require('path'); - -/** - * @type {import("puppeteer").Configuration} - */ -module.exports = { - // Changes the cache location for Puppeteer. - cacheDirectory: join(__dirname, 'node_modules', '.cache', 'puppeteer'), -}; diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md index c63af734ae66fa..e452526ecf3d55 100644 --- a/CODE_OF_CONDUCT.md +++ b/CODE_OF_CONDUCT.md @@ -39,7 +39,7 @@ This Code of Conduct applies within all community spaces, and also applies when ## Enforcement -Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at i@diygod.me. All complaints will be reviewed and investigated promptly and fairly. +Instances of abusive, harassing, or otherwise unacceptable behavior may be reported to the community leaders responsible for enforcement at . All complaints will be reviewed and investigated promptly and fairly. All community leaders are obligated to respect the privacy and security of the reporter of any incident. @@ -74,11 +74,11 @@ Community leaders will follow these Community Impact Guidelines in determining t ## Attribution This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 2.0, -available at https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. +available at . Community Impact Guidelines were inspired by [Mozilla's code of conduct enforcement ladder](https://github.com/mozilla/diversity). [homepage]: https://www.contributor-covenant.org For answers to common questions about this code of conduct, see the FAQ at -https://www.contributor-covenant.org/faq. Translations are available at https://www.contributor-covenant.org/translations. +. Translations are available at . diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 963776c93615e5..273aa2c8b304c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,3 +1 @@ -## 请参见[参与我们](https://docs.rsshub.app/zh/joinus/quick-start) - -## Please refer to [Join Us](https://docs.rsshub.app/joinus/quick-start) +## Please refer to [Join Us](https://docs.rsshub.app/joinus/) diff --git a/Dockerfile b/Dockerfile index 0a765325cf8540..9089f0ca955d48 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM node:18-bullseye AS dep-builder +FROM node:22-bookworm AS dep-builder # Here we use the non-slim image to provide build-time deps (compilers and python), thus no need to install later. # This effectively speeds up qemu-based cross-build. @@ -8,6 +8,7 @@ WORKDIR /app ARG USE_CHINA_NPM_REGISTRY=0 RUN \ set -ex && \ + corepack enable pnpm && \ if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \ echo 'use npm mirror' && \ npm config set registry https://registry.npmmirror.com && \ @@ -15,6 +16,7 @@ RUN \ pnpm config set registry https://registry.npmmirror.com ; \ fi; +COPY ./tsconfig.json /app/ COPY ./pnpm-lock.yaml /app/ COPY ./package.json /app/ @@ -22,67 +24,67 @@ COPY ./package.json /app/ RUN \ set -ex && \ export PUPPETEER_SKIP_DOWNLOAD=true && \ - corepack enable pnpm && \ - pnpm install --prod --frozen-lockfile && \ + pnpm install --frozen-lockfile && \ pnpm rb # --------------------------------------------------------------------------------------------------------------------- -FROM debian:bullseye-slim AS dep-version-parser +FROM debian:bookworm-slim AS dep-version-parser # This stage is necessary to limit the cache miss scope. # With this stage, any modification to package.json won't break the build cache of the next two stages as long as the # version unchanged. -# node:18-bullseye-slim is based on debian:bullseye-slim so this stage would not cause any additional download. +# node:22-bookworm-slim is based on debian:bookworm-slim so this stage would not cause any additional download. WORKDIR /ver COPY ./package.json /app/ RUN \ set -ex && \ - grep -Po '(?<="puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version && \ - grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \ - grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version + grep -Po '(?<="puppeteer": ")[^\s"]*(?=")' /app/package.json | tee /ver/.puppeteer_version + # grep -Po '(?<="@vercel/nft": ")[^\s"]*(?=")' /app/package.json | tee /ver/.nft_version && \ + # grep -Po '(?<="fs-extra": ")[^\s"]*(?=")' /app/package.json | tee /ver/.fs_extra_version # --------------------------------------------------------------------------------------------------------------------- -FROM node:18-bullseye-slim AS docker-minifier +FROM node:22-bookworm-slim AS docker-minifier # The stage is used to further reduce the image size by removing unused files. -WORKDIR /minifier -COPY --from=dep-version-parser /ver/* /minifier/ - -ARG USE_CHINA_NPM_REGISTRY=0 -RUN \ - set -ex && \ - if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \ - npm config set registry https://registry.npmmirror.com && \ - yarn config set registry https://registry.npmmirror.com && \ - pnpm config set registry https://registry.npmmirror.com ; \ - fi; \ - corepack enable pnpm && \ - pnpm add @vercel/nft@$(cat .nft_version) fs-extra@$(cat .fs_extra_version) --save-prod +WORKDIR /app +# COPY --from=dep-version-parser /ver/* /minifier/ + +# ARG USE_CHINA_NPM_REGISTRY=0 +# RUN \ +# set -ex && \ +# if [ "$USE_CHINA_NPM_REGISTRY" = 1 ]; then \ +# npm config set registry https://registry.npmmirror.com && \ +# yarn config set registry https://registry.npmmirror.com && \ +# pnpm config set registry https://registry.npmmirror.com ; \ +# fi; \ +# corepack enable pnpm && \ +# pnpm add @vercel/nft@$(cat .nft_version) fs-extra@$(cat .fs_extra_version) --save-prod COPY . /app COPY --from=dep-builder /app /app RUN \ set -ex && \ - cp /app/scripts/docker/minify-docker.js /minifier/ && \ - export PROJECT_ROOT=/app && \ - node /minifier/minify-docker.js && \ - rm -rf /app/node_modules /app/scripts && \ - mv /app/app-minimal/node_modules /app/ && \ - rm -rf /app/app-minimal && \ + # cp /app/scripts/docker/minify-docker.js /minifier/ && \ + # export PROJECT_ROOT=/app && \ + # node /minifier/minify-docker.js && \ + # rm -rf /app/node_modules /app/scripts && \ + # mv /app/app-minimal/node_modules /app/ && \ + # rm -rf /app/app-minimal && \ + npm run build && \ ls -la /app && \ du -hd1 /app # --------------------------------------------------------------------------------------------------------------------- -FROM node:18-bullseye-slim AS chromium-downloader +FROM node:22-bookworm-slim AS chromium-downloader # This stage is necessary to improve build concurrency and minimize the image size. # Yeah, downloading Chromium never needs those dependencies below. WORKDIR /app -COPY ./.puppeteerrc.js /app/ +COPY ./.puppeteerrc.cjs /app/ COPY --from=dep-version-parser /ver/.puppeteer_version /app/.puppeteer_version ARG TARGETPLATFORM @@ -109,12 +111,12 @@ RUN \ # --------------------------------------------------------------------------------------------------------------------- -FROM node:18-bullseye-slim AS app +FROM node:22-bookworm-slim AS app LABEL org.opencontainers.image.authors="https://github.com/DIYgod/RSSHub" -ENV NODE_ENV production -ENV TZ Asia/Shanghai +ENV NODE_ENV=production +ENV TZ=Asia/Shanghai WORKDIR /app @@ -123,14 +125,14 @@ ARG TARGETPLATFORM ARG PUPPETEER_SKIP_DOWNLOAD=1 # https://pptr.dev/troubleshooting#chrome-headless-doesnt-launch-on-unix # https://github.com/puppeteer/puppeteer/issues/7822 -# https://www.debian.org/releases/bullseye/amd64/release-notes/ch-information.en.html#noteworthy-obsolete-packages +# https://www.debian.org/releases/bookworm/amd64/release-notes/ch-information.en.html#noteworthy-obsolete-packages # The official recommended way to use Puppeteer on arm/arm64 is to install Chromium from the distribution repositories: # https://github.com/puppeteer/puppeteer/blob/07391bbf5feaf85c191e1aa8aa78138dce84008d/packages/puppeteer-core/src/node/BrowserFetcher.ts#L128-L131 RUN \ set -ex && \ apt-get update && \ apt-get install -yq --no-install-recommends \ - dumb-init \ + dumb-init git curl \ ; \ if [ "$PUPPETEER_SKIP_DOWNLOAD" = 0 ]; then \ if [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ @@ -144,20 +146,18 @@ RUN \ apt-get install -yq --no-install-recommends \ chromium \ && \ - echo 'CHROMIUM_EXECUTABLE_PATH=chromium' | tee /app/.env ; \ + echo "CHROMIUM_EXECUTABLE_PATH=$(which chromium)" | tee /app/.env ; \ fi; \ fi; \ rm -rf /var/lib/apt/lists/* COPY --from=chromium-downloader /app/node_modules/.cache/puppeteer /app/node_modules/.cache/puppeteer -# if grep matches nothing then it will exit with 1, thus, we cannot `set -e` here RUN \ - set -x && \ + set -ex && \ if [ "$PUPPETEER_SKIP_DOWNLOAD" = 0 ] && [ "$TARGETPLATFORM" = 'linux/amd64' ]; then \ echo 'Verifying Chromium installation...' && \ - ldd $(find /app/node_modules/.cache/puppeteer/ -name chrome -type f) | grep "not found" ; \ - if [ "$?" = 0 ]; then \ + if ldd $(find /app/node_modules/.cache/puppeteer/ -name chrome -type f) | grep "not found"; then \ echo "!!! Chromium has unmet shared libs !!!" && \ exit 1 ; \ else \ diff --git a/Procfile b/Procfile deleted file mode 100644 index a9472b4a38e23e..00000000000000 --- a/Procfile +++ /dev/null @@ -1 +0,0 @@ -web: node lib/index.js diff --git a/README.md b/README.md index 53ca3bfe23cab7..defee8667f8dac 100644 --- a/README.md +++ b/README.md @@ -3,94 +3,58 @@

RSSHub

-> 🍰 Everything is RSSible +> 🧡 Everything is RSSible -[![telegram](https://img.shields.io/badge/chat-telegram-brightgreen.svg?logo=telegram&style=flat-square)](https://t.me/rsshub) -[![npm publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/npm-publish.yml?branch=master&label=npm%20publish&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub) -[![docker publish](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/docker-release.yml?branch=master&label=docker%20publish&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub) +[![](https://img.shields.io/badge/dynamic/json?url=https://rsshub-analytics.diygod.workers.dev/&query=requests&color=F38020&label=requests&logo=cloudflare&style=flat-square&suffix=/month)](https://rsshub.app) +[![docker publish](https://img.shields.io/docker/pulls/diygod/rsshub?label=docker%20pulls&logo=docker&style=flat-square)](https://hub.docker.com/r/diygod/rsshub) +[![npm publish](https://img.shields.io/npm/dt/rsshub?label=npm%20downloads&logo=npm&style=flat-square)](https://www.npmjs.com/package/rsshub) [![test](https://img.shields.io/github/actions/workflow/status/DIYgod/RSSHub/test.yml?branch=master&label=test&logo=github&style=flat-square)](https://github.com/DIYgod/RSSHub/actions/workflows/test.yml?query=event%3Apush+branch%3Amaster) [![Test coverage](https://img.shields.io/codecov/c/github/DIYgod/RSSHub.svg?style=flat-square&logo=codecov)](https://app.codecov.io/gh/DIYgod/RSSHub/branch/master) -[![CodeFactor](https://www.codefactor.io/repository/github/diygod/rsshub/badge)](https://www.codefactor.io/repository/github/diygod/rsshub) -[![DeepScan grade](https://deepscan.io/api/teams/6244/projects/8135/branches/92448/badge/grade.svg)](https://deepscan.io/dashboard#view=project&tid=6244&pid=8135&bid=92448) +[![Visitors](https://hits.seeyoufarm.com/api/count/incr/badge.svg?url=https%3A%2F%2Fgithub.com%2FDIYgod%2FRSSHub&count_bg=%23FF752E&title_bg=%23555555&icon=rss.svg&icon_color=%23FF752E&title=RSS+lovers&edge_flat=true)](https://github.com/DIYgod/RSSHub) + +[![Telegram group](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.swo.moe%2Fstats%2Ftelegram%2Frsshub&query=count&color=2CA5E0&label=Telegram%20Group&logo=telegram&cacheSeconds=3600&style=flat-square)](https://t.me/rsshub) [![Telegram channel](https://img.shields.io/badge/dynamic/json?url=https%3A%2F%2Fapi.swo.moe%2Fstats%2Ftelegram%2FawesomeRSSHub&query=count&color=2CA5E0&label=Telegram%20Channel&logo=telegram&cacheSeconds=3600&style=flat-square)](https://t.me/awesomeRSSHub) [![X (Twitter)](https://img.shields.io/badge/any_text-Follow-blue?color=2CA5E0&label=Twitter&logo=X&cacheSeconds=3600&style=flat-square)](https://x.com/intent/follow?screen_name=_RSSHub) ## Introduction -RSSHub is an open source, easy to use, and extensible RSS feed generator. It's capable of generating RSS feeds from pretty much everything. +RSSHub is the world's largest RSS network, consisting of over 5,000 global instances. RSSHub delivers millions of contents aggregated from all kinds of sources, our vibrant open source community is ensuring the deliver of RSSHub's new routes, new features and bug fixes. -RSSHub can be used with browser extension [RSSHub Radar](https://github.com/DIYgod/RSSHub-Radar) and mobile auxiliary app [RSSBud](https://github.com/Cay-Zhang/RSSBud) (iOS) and [RSSAid](https://github.com/LeetaoGoooo/RSSAid) (Android) - -[English docs](https://docs.rsshub.app) | [Telegram Group](https://t.me/rsshub) | [Telegram Channel](https://t.me/awesomeRSSHub) - -[中文文档](https://docs.rsshub.app/zh/) | [Telegram 群](https://t.me/rsshub) | [Telegram 频道](https://t.me/awesomeRSSHub) - -## Special Thanks - -### Special Sponsors - -

- -

- -[![](https://opencollective.com/static/images/become_sponsor.svg)](https://docs.rsshub.app/support/) - -### Contributors - -[![](https://opencollective.com/RSSHub/contributors.svg?width=890)](https://github.com/DIYgod/RSSHub/graphs/contributors) - -Logo designer [sheldonrrr](https://dribbble.com/sheldonrrr) - -### Backers - -        +[Documentation](https://docs.rsshub.app) | [Telegram Group](https://t.me/rsshub) | [Telegram Channel](https://t.me/awesomeRSSHub) | [X (Twitter)](https://x.com/intent/follow?screen_name=_RSSHub) ## Related Projects - [RSSHub Radar](https://github.com/DIYgod/RSSHub-Radar) | A browser extension that can help you quickly discover and subscribe to the RSS and RSSHub of current websites. -- [RSSBud](https://github.com/Cay-Zhang/RSSBud) ([TestFlight](https://testflight.apple.com/join/rjCVzzHP)) | RSSHub Radar for iOS platform, designed specifically for mobile ecosystem optimization. +- [RSSBud](https://github.com/Cay-Zhang/RSSBud) | RSSHub Radar for iOS platform, designed specifically for mobile ecosystem optimization. - [RSSAid](https://github.com/LeetaoGoooo/RSSAid) | RSSHub Radar for Android platform built with Flutter. - [DocSearch](https://github.com/Fatpandac/DocSearch) | Link RSSHub DocSearch into Raycast -## Join Us +## Contribute We welcome all pull requests. Suggestions and feedback are also welcomed [here](https://github.com/DIYgod/RSSHub/issues). -Refer to [Join Us](https://docs.rsshub.app/joinus/quick-start) +Refer to [Quick Start](https://docs.rsshub.app/joinus/) ## Deployment -Refer to [Deployment](https://docs.rsshub.app/install/) - -## Support RSSHub - -Refer to [Support RSSHub](https://docs.rsshub.app/support/) +Refer to [Deployment](https://docs.rsshub.app/deploy/) -RSSHub is open source and completely free under the MIT license. However, just like any other open source project, as the project grows, the hosting, development and maintenance requires funding support. - -You can support RSSHub via donations. - -### Recurring Donation +## Special Thanks -Recurring donors will be rewarded via express issue response, or even have your name displayed on our GitHub page and website. +
-- Become a Sponser on [GitHub](https://github.com/sponsors/DIYgod) -- Become a Sponser on [Patreon](https://www.patreon.com/DIYgod) -- Become a Sponser on [Open Collective](https://opencollective.com/RSSHub) -- Become a Sponser on [爱发电](https://afdian.net/@diygod) -- Contact us directly: i@diygod.me +[![](https://opencollective.com/RSSHub/contributors.svg?width=890)](https://github.com/DIYgod/RSSHub/graphs/contributors) -### One-time Donation +Logo designer [sheldonrrr](https://dribbble.com/sheldonrrr) -We accept donations via the following ways: +[![](https://raw.githubusercontent.com/DIYgod/sponsors/main/sponsors.simple.svg)](https://github.com/DIYgod/sponsors) -- [WeChat Pay](https://archive.diygod.me/images/wx.jpg) -- [Alipay](https://archive.diygod.me/images/zfb.jpg) -- [Paypal](https://www.paypal.me/DIYgod) +               +
## Author **RSSHub** © [DIYgod](https://github.com/DIYgod), Released under the [MIT](./LICENSE) License.
Authored and maintained by DIYgod with help from contributors ([list](https://github.com/DIYgod/RSSHub/contributors)). -> Blog [@DIYgod](https://diygod.cc) · GitHub [@DIYgod](https://github.com/DIYgod) · Twitter [@DIYgod](https://twitter.com/DIYgod) · Telegram Channel [@awesomeDIYgod](https://t.me/awesomeDIYgod) +> Blog [@DIYgod](https://diygod.cc) · GitHub [@DIYgod](https://github.com/DIYgod) · X (Twitter) [@DIYgod](https://x.com/DIYgod) · Telegram Channel [@awesomeDIYgod](https://t.me/awesomeDIYgod) diff --git a/SECURITY.md b/SECURITY.md index d7e04b96a72eec..97af6228f31b85 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -6,4 +6,4 @@ Latest commits in master branch ## Reporting a Vulnerability -If you believe you have found a security vulnerability in RSSHub, please let us know right away, you can email us at [i@diygod.me](mailto:i@diygod.me). We will investigate all legitimate reports and do our best to quickly fix the problem. +If you believe you have found a security vulnerability in RSSHub, please let us know right away, you can [open a draft security advisory](https://github.com/DIYgod/RSSHub/security/advisories/new) or email us at [i@diygod.me](mailto:i@diygod.me). We will investigate all legitimate reports and do our best to quickly fix the problem. diff --git a/api/vercel.js b/api/vercel.js deleted file mode 100644 index 02ad653ab617f4..00000000000000 --- a/api/vercel.js +++ /dev/null @@ -1,14 +0,0 @@ -const path = require('path'); -const moduleAlias = require('module-alias'); -moduleAlias.addAlias('@', path.join(__dirname, '../lib')); - -const config = require('../lib/config'); -config.set({ - NO_LOGFILES: true, -}); - -const app = require('../lib/app'); - -module.exports = (req, res) => { - app.callback()(req, res); -}; diff --git a/api/vercel.ts b/api/vercel.ts new file mode 100644 index 00000000000000..12a2a910b723a3 --- /dev/null +++ b/api/vercel.ts @@ -0,0 +1,17 @@ +const path = require('path'); +const moduleAlias = require('module-alias'); +moduleAlias.addAlias('@', path.join(__dirname, '../lib')); + +const { setConfig } = require('../lib/config'); +setConfig({ + NO_LOGFILES: true, +}); + +const { handle } = require('hono/vercel'); +const app = require('../lib/app'); +const logger = require('../lib/utils/logger'); + +logger.info(`🎉 RSSHub is running! Cheers!`); +logger.info('💖 Can you help keep this open source project alive? Please sponsor 👉 https://docs.rsshub.app/sponsor'); + +module.exports = handle(app); diff --git a/app.json b/app.json index 796f14bb931b84..e351d522029245 100644 --- a/app.json +++ b/app.json @@ -1,6 +1,6 @@ { "name": "RSSHub", - "description": "万物皆可 RSS", + "description": "Everything is RSSible", "repository": "https://github.com/DIYgod/RSSHub", "website": "https://docs.rsshub.app/", "logo": "https://i.loli.net/2019/04/23/5cbeb7e41414c.png", diff --git a/assets/404.html b/assets/404.html index f8c64279428b4f..2c6b99e4c5a94f 100644 --- a/assets/404.html +++ b/assets/404.html @@ -1,4 +1,4 @@ \ No newline at end of file + diff --git a/assets/CNAME b/assets/CNAME index 5ab3a9991b7eef..7bec7fdc352637 100644 --- a/assets/CNAME +++ b/assets/CNAME @@ -1 +1 @@ -rsshub.js.org \ No newline at end of file +rsshub.js.org diff --git a/assets/index.html b/assets/index.html index f8c64279428b4f..2c6b99e4c5a94f 100644 --- a/assets/index.html +++ b/assets/index.html @@ -1,4 +1,4 @@ \ No newline at end of file + diff --git a/assets/radar-rules.js b/assets/radar-rules.js deleted file mode 100644 index bb0034b05f788f..00000000000000 --- a/assets/radar-rules.js +++ /dev/null @@ -1,483 +0,0 @@ -({ - 'aamacau.com': { _name: '論盡媒體 AllAboutMacau Media', '.': [{ title: '话题', docs: 'https://docs.rsshub.app/routes/new-media#lun-jin-mei-ti-allaboutmacau-media-hua-ti', source: ['/'], target: '/:category?/:id?' }] }, - 'eprice.com.tw': { _name: 'ePrice', '.': [{ title: 'ePrice 比價王', docs: 'https://docs.rsshub.app/routes/new-media#eprice', source: ['/'], target: '/:region?' }] }, - 'eprice.com.hk': { _name: 'ePrice', '.': [{ title: 'ePrice 香港', docs: 'https://docs.rsshub.app/routes/new-media#eprice', source: ['/'], target: '/:region?' }] }, - 'furstar.jp': { - _name: 'Furstar', - '.': [ - { title: '最新售卖角色列表', docs: 'https://docs.rsshub.app/routes/shopping#furstar-zui-xin-shou-mai-jiao-se-lie-biao', source: ['/:lang', '/'], target: '/furstar/characters/:lang' }, - { title: '已经出售的角色列表', docs: 'https://docs.rsshub.app/routes/shopping#furstar-yi-jing-chu-shou-de-jiao-se-lie-biao', source: ['/:lang/archive.php', '/archive.php'], target: '/furstar/archive/:lang' }, - { title: '画师列表', docs: 'https://docs.rsshub.app/routes/shopping#furstar-hua-shi-lie-biao', source: ['/'], target: '/furstar/artists' }, - ], - }, - 'trow.cc': { _name: 'The Ring of Wonder', '.': [{ title: '首页更新', docs: 'https://docs.rsshub.app/routes/bbs#the-ring-of-wonder', source: ['/'], target: '/portal' }] }, - 'github.com': { - _name: 'GitHub', - '.': [ - { title: '用户仓库', docs: 'https://docs.rsshub.app/routes/programming#github', source: '/:user', target: '/github/repos/:user' }, - { title: '用户 Followers', docs: 'https://docs.rsshub.app/routes/programming#github', source: '/:user', target: '/github/user/followers/:user' }, - { title: 'Trending', docs: 'https://docs.rsshub.app/routes/programming#github', source: '/trending', target: '/github/trending/:since' }, - { title: 'Trending', docs: 'https://docs.rsshub.app/routes/programming#github', source: '/topics', target: '/github/topics/:name/:qs?' }, - { title: '仓库 Issue', docs: 'https://docs.rsshub.app/routes/programming#github', source: ['/:user/:repo/issues', '/:user/:repo/issues/:id', '/:user/:repo'], target: '/github/issue/:user/:repo' }, - { title: '仓库 Pull Requests', docs: 'https://docs.rsshub.app/routes/programming#github', source: ['/:user/:repo/pulls', '/:user/:repo/pulls/:id', '/:user/:repo'], target: '/github/pull/:user/:repo' }, - { title: '仓库 Stars', docs: 'https://docs.rsshub.app/routes/programming#github', source: ['/:user/:repo/stargazers', '/:user/:repo'], target: '/github/stars/:user/:repo' }, - { title: '仓库 Branches', docs: 'https://docs.rsshub.app/routes/programming#github', source: ['/:user/:repo/branches', '/:user/:repo'], target: '/github/branches/:user/:repo' }, - { title: '文件 Commits', docs: 'https://docs.rsshub.app/routes/programming#github', source: '/:user/:repo/blob/:branch/*filepath', target: '/github/file/:user/:repo/:branch/:filepath' }, - { title: '用户 Starred Repositories', docs: 'https://docs.rsshub.app/routes/programming#github', source: '/:user', target: '/github/starred_repos/:user' }, - { title: '仓库 Contributors', docs: 'https://docs.rsshub.app/routes/programming#github', source: ['/:user/:repo/graphs/contributors', '/:user/:repo'], target: '/github/contributors/:user/:repo' }, - ], - }, - 'algocasts.io': { _name: 'AlgoCasts', '.': [{ title: '视频更新', docs: 'https://docs.rsshub.app/routes/programming#algocasts', source: '/episodes', target: '/algocasts' }] }, - 'soulapp.cn': { _name: 'Soul', '.': [{ title: '瞬间更新', docs: 'https://docs.rsshub.app/routes/social-media#soul' }] }, - 'anime1.me': { - _name: 'Anime1', - '.': [ - { title: '動畫', docs: 'https://docs.rsshub.app/routes/anime#anime1', source: '/category/:time/:name', target: '/anime1/anime/:time/:name' }, - { - title: '搜尋', - docs: 'https://docs.rsshub.app/routes/anime#anime1', - source: '/', - target: (params, url) => { - const keyword = new URL(url).searchParams.get('s'); - return keyword ? `/anime1/search/${keyword}` : ''; - }, - }, - ], - }, - 'swufe.edu.cn': { - _name: '西南财经大学', - it: [ - { title: '经济信息工程学院 - 通知公告', docs: 'https://docs.rsshub.app/routes/university#xi-nan-cai-jing-da-xue', source: '/index/tzgg.htm', target: '/swufe/seie/tzgg' }, - { title: '经济信息工程学院 - 学院新闻', docs: 'https://docs.rsshub.app/routes/university#xi-nan-cai-jing-da-xue', source: '/index/xyxw.htm', target: '/swufe/seie/xyxw' }, - ], - }, - 'ishuhui.com': { _name: '鼠绘漫画', www: [{ title: '鼠绘漫画', docs: 'https://docs.rsshub.app/routes/anime#shu-hui-man-hua', source: '/comics/anime/:id', target: '/shuhui/comics/:id' }] }, - 'www.chicagotribune.com': { _name: 'Chicago Tribune', www: [{ title: 'Chicago Tribune', docs: 'https://docs.rsshub.app/routes/traditional_media#chicago-tribune', source: '/' }] }, - 'haimaoba.com': { _name: '海猫吧', www: [{ title: '漫画更新', docs: 'https://docs.rsshub.app/routes/anime#hai-mao-ba', source: '/catalog/:id', target: '/haimaoba/:id' }] }, - 'pgyer.com': { _name: '蒲公英应用分发', www: [{ title: 'app更新', docs: 'https://docs.rsshub.app/routes/program-update#pu-gong-ying-ying-yong-fen-fa', source: '/:app', target: '/pgyer/:app' }] }, - 'wineyun.com': { _name: '酒云网', www: [{ title: '最新商品', docs: 'https://docs.rsshub.app/routes/other#jiu-yun-wang', source: ['/:category'], target: '/wineyun/:category' }] }, - 'playstation.com': { - _name: 'PlayStation', - store: [ - { title: '游戏列表', docs: 'https://docs.rsshub.app/routes/game#playstation', source: '/zh-hans-hk/grid/:id/:page', target: '/ps/list/:id' }, - { title: '折扣|价格', docs: 'https://docs.rsshub.app/routes/game#playstation', source: ['/:lang/product/:gridName'], target: '/ps/:lang/product/:gridName' }, - ], - www: [ - { title: '用户奖杯', docs: 'https://docs.rsshub.app/routes/game#playstation' }, - { title: '系统更新纪录', docs: 'https://docs.rsshub.app/routes/game#playstation' }, - ], - }, - 'monsterhunter.com': { - _name: '怪物猎人世界', - www: [ - { title: '更新情报', docs: 'https://docs.rsshub.app/routes/game#guai-wu-lie-ren-shi-jie', source: ['', '/*tpath'], target: '/mhw/update' }, - { title: '最新消息', docs: 'https://docs.rsshub.app/routes/game#guai-wu-lie-ren-shi-jie', source: ['', '/*tpath'], target: '/mhw/news' }, - ], - }, - 'vgtime.com': { - _name: '游戏时光', - www: [ - { title: '新闻', docs: 'https://docs.rsshub.app/routes/game#you-xi-shi-guang', source: '/topic/index.', target: '/vgtime/news' }, - { title: '游戏发售表', docs: 'https://docs.rsshub.app/routes/game#you-xi-shi-guang', source: '/game/release.', target: '/vgtime/release' }, - { title: '关键词资讯', docs: 'https://docs.rsshub.app/routes/game#you-xi-shi-guang', source: '/search/list.', target: (params, url) => `/vgtime/keyword/${new URL(url).searchParams.get('keyword')}` }, - ], - }, - 'bing.com': { _name: 'Bing', www: [{ title: '每日壁纸', docs: 'https://docs.rsshub.app/routes/picture#bing-bi-zhi', source: '', target: '/bing' }] }, - 'wegene.com': { - _name: 'WeGene', - www: [ - { title: '最近更新', docs: 'https://docs.rsshub.app/routes/other#wegene', source: '', target: '/wegene/newest' }, - { title: '栏目', docs: 'https://docs.rsshub.app/routes/other#wegene', source: '/crowdsourcing', target: '/wegene/column/all/all' }, - ], - }, - '3ycy.com': { _name: '三界异次元', www: [{ title: '最近更新', docs: 'https://docs.rsshub.app/routes/anime#san-jie-yi-ci-yuan', source: '/', target: '/3ycy/home' }] }, - 'emi-nitta.net': { - _name: 'Emi Nitta', - '.': [ - { title: '最近更新', docs: 'https://docs.rsshub.app/routes/other#xin-tian-hui-hai-guan-fang-wang-zhan', source: '/updates', target: '/emi-nitta/updates' }, - { title: '新闻', docs: 'https://docs.rsshub.app/routes/other#xin-tian-hui-hai-guan-fang-wang-zhan', source: '/contents/news', target: '/emi-nitta/news' }, - ], - }, - 'alter-shanghai.cn': { _name: 'Alter', '.': [{ title: '新闻', docs: 'https://docs.rsshub.app/routes/shopping#alter-zhong-guo', source: '/cn/news', target: '/alter-cn/news' }] }, - 'itslide.com': { _name: 'ITSlide', www: [{ title: '最新', docs: 'https://docs.rsshub.app/routes/programming#itslide', source: '/*', target: '/itslide/new' }] }, - 'leboncoin.fr': { _name: 'leboncoin', www: [{ title: 'ads', docs: 'https://docs.rsshub.app/routes/shopping#leboncoin', source: '/recherche', target: (params, url) => '/leboncoin/ad/' + url.split('?')[1] }] }, - 'yuancheng.work': { - _name: '远程.work', - '.': [ - { - title: '招聘信息', - docs: 'https://docs.rsshub.app/routes/other#yuan-cheng-work', - source: '/:caty', - target: (params, url) => { - if (!url) { - return '/remote-work'; - } - return '/remote-work/' + /\w+-(\w+)-\w+/.exec(url)[1]; - }, - }, - ], - }, - 'chinatimes.com': { _name: '中時電子報', www: [{ title: '新聞', docs: 'https://docs.rsshub.app/routes/traditional-media#zhong-shi-dian-zi-bao', source: '/:caty', target: (params) => '/chinatimes/' + params.caty }] }, - 'govopendata.com': { _name: '新闻联播文字版', cn: [{ title: '新闻联播文字版', docs: 'https://docs.rsshub.app/routes/traditional-media#xin-wen-lian-bo-wen-zi-ban', source: '/xinwenlianbo', target: '/xinwenlianbo/index' }] }, - 'steampowered.com': { _name: 'Steam', store: [{ title: 'search', docs: 'https://docs.rsshub.app/routes/game#steam', source: '/search/', target: (params, url) => `/steam/search/${new URL(url).searchParams}` }] }, - 'xiaomi.cn': { _name: '小米社区', www: [{ title: '圈子', docs: 'https://docs.rsshub.app/routes/bbs#xiao-mi-she-qu', source: '/board/:boardId', target: '/mi/bbs/board/:boardId' }] }, - 'suzhou.gov.cn': { _name: '苏州市政府', www: [{ title: '政府新闻', docs: 'https://docs.rsshub.app/routes/government#su-zhou-shi-ren-min-zheng-fu', source: '/szsrmzf/:uid/nav_list.', target: '/gov/suzhou/news/:uid' }] }, - 'mqube.net': { - _name: 'MQube', - www: [ - { title: '全站最近更新', docs: 'https://docs.rsshub.app/routes/multimedia#mqube', source: '/', target: '/mqube/latest' }, - { title: '全站每日排行', docs: 'https://docs.rsshub.app/routes/multimedia#mqube', source: '/', target: '/mqube/top' }, - { title: '个人最近更新', docs: 'https://docs.rsshub.app/routes/multimedia#mqube', source: '/user/:user', target: '/mqube/user/:user' }, - { title: '标签最近更新', docs: 'https://docs.rsshub.app/routes/multimedia#mqube', source: '/search/tag/:tag', target: '/mqube/tag/:tag' }, - ], - }, - 'last.fm': { - _name: 'Last.fm', - www: [ - { title: '用户播放记录', docs: 'https://docs.rsshub.app/routes/multimedia#last-fm', source: ['/user/:user', '/user/:user/*'], target: '/lastfm/recent/:user' }, - { title: '用户 Love 记录', docs: 'https://docs.rsshub.app/routes/multimedia#last-fm', source: ['/user/:user', '/user/:user/*'], target: '/lastfm/loved/:user' }, - { title: '站内 Top 榜单', docs: 'https://docs.rsshub.app/routes/multimedia#last-fm', source: '/charts', target: '/lastfm/top' }, - ], - }, - 'ddrk.me': { - _name: '低端影视', - www: [ - { title: '首页', docs: 'https://docs.rsshub.app/routes/multimedia#di-duan-ying-shi', source: '/', target: '/ddrk/index' }, - { title: '标签', docs: 'https://docs.rsshub.app/routes/multimedia#di-duan-ying-shi', source: '/tag/:tag', target: '/ddrk/tag/:tag' }, - { title: '分类', docs: 'https://docs.rsshub.app/routes/multimedia#di-duan-ying-shi', source: ['/category/:category', '/category/:uplevel/:category'], target: '/ddrk/category/:category' }, - { - title: '影视剧集更新', - docs: 'https://docs.rsshub.app/routes/multimedia#di-duan-ying-shi', - source: ['/:name', '/:name/:season'], - target: (params) => { - if (params.name !== 'category' && params.name !== 'tag' && params.name !== 'ddrklogin' && params.name !== 'about' && params.name !== 'deleted') { - return `/ddrk/update/${params.name}${params.season ? '/' + params.season : ''}`; - } - }, - }, - ], - }, - 'hackerone.com': { _name: 'HackerOne', '.': [{ title: 'HackerOne Hacker Activity', docs: 'https://docs.rsshub.app/routes/other#hackerone-hacker-activity', source: '/hacktivity', target: '/hackerone/hacktivity' }] }, - 'cowlevel.net': { _name: '奶牛关', '.': [{ title: '元素文章', docs: 'https://docs.rsshub.app/routes/game#nai-niu-guan', source: ['/element/:id', '/element/:id/article'], target: '/cowlevel/element/:id' }] }, - 'beijing.gov.cn': { wjw: [{ title: '北京卫生健康委员会', docs: 'https://docs.rsshub.app/routes/government#bei-jing-shi-wei-sheng-jian-kang-wei-yuan-hui', source: '/xwzx_20031/:caty', target: '/gov/beijing/mhc/:caty' }] }, - 'ynu.edu.cn': { - _name: '云南大学', - home: [{ title: '官网消息通告', docs: 'https://docs.rsshub.app/routes/university#yun-nan-da-xue', source: '/tzgg.htm', target: '/ynu/home' }], - jwc: [ - { title: '教务处教务科通知', docs: 'https://docs.rsshub.app/routes/university#yun-nan-da-xue', source: '/*', target: '/jwc/1' }, - { title: '教务处学籍科通知', docs: 'https://docs.rsshub.app/routes/university#yun-nan-da-xue', source: '/*', target: '/jwc/2' }, - { title: '教务处教学研究科通知', docs: 'https://docs.rsshub.app/routes/university#yun-nan-da-xue', source: '/*', target: '/jwc/3' }, - { title: '教务处实践科学科通知', docs: 'https://docs.rsshub.app/routes/university#yun-nan-da-xue', source: '/*', target: '/jwc/4' }, - ], - grs: [{ title: '研究生院通知', docs: 'https://docs.rsshub.app/routes/university#yun-nan-da-xue', source: '/*', target: '' }], - }, - // 'biquge5200.com': { www: [{ title: 'biquge5200.com', docs: 'https://docs.rsshub.app/routes/reading#bi-qu-ge-biquge5200-com', source: '/:id', target: '/novel/biquge/:id' }] }, - // 'biquge.info': { www: [{ title: 'biquge.info', docs: 'https://docs.rsshub.app/routes/reading#bi-qu-ge-biquge-info', source: '/:id', target: '/novel/biqugeinfo/:id' }] }, - 'matters.news': { - _name: 'Matters', - '.': [ - { title: '最新排序', docs: 'https://docs.rsshub.app/routes/new-media#matters', source: '', target: '/matters/latest' }, - { title: '标签', docs: 'https://docs.rsshub.app/routes/new-media#matters', source: '/tags/:tid', target: '/matters/tags/:tid' }, - { - title: '作者', - docs: 'https://docs.rsshub.app/routes/new-media#matters', - source: ['/:id', '/:id/comments'], - target: (params) => { - const uid = params.id.replace('@', ''); - return uid ? `/matters/author/${uid}` : ''; - }, - }, - ], - }, - 'zhaishuyuan.com': { _name: '斋书苑', '.': [{ title: '最新章节', docs: 'https://docs.rsshub.app/routes/reading#zhai-shu-yuan', source: ['/book/:id', '/read/:id'], target: '/novel/zhaishuyuan/:id' }] }, - 'hbut.edu.cn': { - _name: '湖北工业大学', - www: [ - { - title: '新闻中心', - docs: 'http://docs.rsshub.app/routes/university#hu-bei-gong-ye-da-xue', - source: '/xwzx/:name', - target: (params) => { - const type = params.name.replace('.htm', ''); - return type ? `/hbut/news/${type}` : '/hbut/news/tzgg'; - }, - }, - ], - jsjxy: [ - { title: '新闻动态', docs: 'http://docs.rsshub.app/routes/university#hu-bei-gong-ye-da-xue', source: '/index/xwdt.htm', target: '/hbut/cs/xwdt' }, - { title: '通知公告', docs: 'http://docs.rsshub.app/routes/university#hu-bei-gong-ye-da-xue', source: '/index/tzgg.htm', target: '/hbut/cs/tzgg' }, - { title: '教学信息', docs: 'http://docs.rsshub.app/routes/university#hu-bei-gong-ye-da-xue', source: '/jxxx.htm', target: '/hbut/cs/jxxx' }, - { title: '科研动态', docs: 'http://docs.rsshub.app/routes/university#hu-bei-gong-ye-da-xue', source: '/kxyj/kydt.htm', target: '/hbut/cs/kydt' }, - { title: '党建活动', docs: 'http://docs.rsshub.app/routes/university#hu-bei-gong-ye-da-xue', source: '/djhd/djhd.htm', target: '/hbut/cs/djhd' }, - ], - }, - 'zhuixinfan.com': { _name: '追新番日剧站', '.': [{ title: '更新列表', docs: 'https://docs.rsshub.app/routes/multimedia#zhui-xin-fan-ri-ju-zhan', source: ['/main.php'], target: '/zhuixinfan/list' }] }, - 'etoland.co.kr': { - _name: 'eTOLAND', - '.': [{ title: '主题贴', docs: 'https://docs.rsshub.app/routes/bbs#etoland', source: ['/bbs/board.php', '/plugin/mobile/board.php'], target: (params, url) => `/etoland/${new URL(url).searchParams.get('bo_table')}` }], - }, - 'onejav.com': { - _name: 'OneJAV BT', - '.': [ - { - title: '今日种子', - docs: 'https://docs.rsshub.app/routes/multimedia#onejav', - source: '/', - target: (params, url, document) => { - const today = document.querySelector('div.card.mb-1.card-overview').getAttribute('data-date').replace(/-/g, ''); - return `/onejav/day/${today}`; - }, - }, - { - title: '今日演员', - docs: 'https://docs.rsshub.app/routes/multimedia#onejav', - source: '/', - target: (params, url, document) => { - const star = document.querySelector('div.card-content > div > a').getAttribute('href'); - return `/onejav${star}`; - }, - }, - { - title: '页面种子', - docs: 'https://docs.rsshub.app/routes/multimedia#onejav', - source: ['/:type', '/:type/:key', '/:type/:key/:morekey'], - target: (params, url, document) => { - const itype = params.morekey === undefined ? params.type : params.type === 'tag' ? 'tag' : 'day'; - let ikey = `${itype === 'day' ? params.type : ''}${params.key || ''}${itype === 'tag' && params.morekey !== undefined ? '%2F' : ''}${params.morekey || ''}`; - if (ikey === '' && itype === 'tag') { - ikey = document.querySelector('div.thumbnail.is-inline > a').getAttribute('href').replace('/tag/', '').replace('/', '%2F'); - } else if (ikey === '' && itype === 'actress') { - ikey = document.querySelector('div.card > a').getAttribute('href').replace('/actress/', ''); - } - return `/onejav/${itype}/${ikey}`; - }, - }, - ], - }, - 'sexinsex.net': { - _name: 'sexinsex', - '.': [ - { - title: '分区帖子', - docs: 'https://docs.rsshub.app/routes/multimedia#sexinsex', - source: '/bbs/:path', - target: (params, url) => { - let pid, typeid; - const static_matched = params.path.match(/forum-(\d+)-\d+.html/); - if (static_matched) { - pid = static_matched[1]; - } else if (params.path === 'forumdisplay.php') { - pid = new URL(url).searchParams.get('fid'); - typeid = new URL(url).searchParams.get('typeid'); - } else { - return false; - } - return `/sexinsex/${pid}/${typeid ? typeid : ''}`; - }, - }, - ], - }, - 't66y.com': { - _name: '草榴社区', - www: [ - { - title: '分区帖子', - docs: 'https://docs.rsshub.app/routes/multimedia#cao-liu-she-qu', - source: '/thread0806.php', - target: (params, url) => { - const id = new URL(url).searchParams.get('fid'); - const type = new URL(url).searchParams.get('type'); - return `/t66y/${id}/${type ? type : ''}`; - }, - }, - ], - }, - 'umass.edu': { - _name: 'UMASS Amherst', - ece: [ - { title: 'ECE News', docs: 'http://docs.rsshub.app/routes/university#umass-amherst', source: '/news', target: '/umass/amherst/ecenews' }, - { title: 'ECE Seminar', docs: 'http://docs.rsshub.app/routes/university#umass-amherst', source: '/seminars', target: '/umass/amherst/eceseminar' }, - ], - 'www.cics': [{ title: 'CICS News', docs: 'http://docs.rsshub.app/routes/university#umass-amherst', source: '/news', target: '/umass/amherst/csnews' }], - www: [ - { title: 'IPO Events', docs: 'http://docs.rsshub.app/routes/university#umass-amherst', source: '/ipo/iss/events', target: '/umass/amherst/ipoevents' }, - { title: 'IPO Featured Stories', docs: 'http://docs.rsshub.app/routes/university#umass-amherst', source: '/ipo/iss/featured-stories', target: '/umass/amherst/ipostories' }, - ], - }, - 'bjeea.com': { - _name: '北京考试院', - www: [ - { title: '首页 / 通知公告', docs: 'https://docs.rsshub.app/routes/government#bei-jing-jiao-yu-kao-shi-yuan', source: ['/bjeeagg'], target: '/gov/beijing/bjeea/bjeeagg' }, - { title: '首页 / 招考政策', docs: 'https://docs.rsshub.app/routes/government#bei-jing-jiao-yu-kao-shi-yuan', source: ['/zkzc'], target: '/gov/beijing/bjeea/zkzc' }, - { title: '首页 / 自考快递', docs: 'https://docs.rsshub.app/routes/government#bei-jing-jiao-yu-kao-shi-yuan', source: ['/zkkd'], target: '/gov/beijing/bjeea/zkkd' }, - ], - }, - 'ems.com.cn': { _name: '中国邮政速递物流', www: [{ title: '新闻', docs: 'https://docs.rsshub.app/routes/other#zhong-guo-you-zheng-su-di-wu-liu', source: '/aboutus/xin_wen_yu_shi_jian', target: '/ems/news' }] }, - 'popiapp.cn': { - _name: 'Popi 提问箱', - www: [ - { - title: '提问箱新回答', - docs: 'https://docs.rsshub.app/routes/social-media#popi-ti-wen-xiang', - source: '/:id', - target: (params) => { - if (params.id) { - return '/popiask/:id'; - } - }, - }, - ], - }, - 'nppa.gov.cn': { - _name: '国家新闻出版署', - www: [ - { title: '栏目', docs: 'https://docs.rsshub.app/routes/government#guo-jia-xin-wen-chu-ban-shu', source: '/nppa/channels/:channel', target: (params, url) => `/gov/nppa/${/nppa\/channels\/(\d+)\./.exec(url)[1]}` }, - { - title: '内容', - docs: 'https://docs.rsshub.app/routes/government#guo-jia-xin-wen-chu-ban-shu', - source: '/nppa/contents/:channel/:content', - target: (params, url) => `/gov/nppa/${/nppa\/contents\/(\d+\/\d+)\.shtml/.exec(url)[1]}`, - }, - ], - }, - 'jjmhw.cc': { _name: '漫小肆', www: [{ title: '漫画更新', docs: 'https://docs.rsshub.app/routes/anime#man-xiao-si', source: '/book/:id', target: '/manxiaosi/book/:id' }] }, - 'wenxuecity.com': { - _name: '文学城', - blog: [ - { title: '博客', docs: 'https://docs.rsshub.app/routes/bbs#wen-xue-cheng-bo-ke', source: '/myblog/:id', target: '/wenxuecity/blog/:id' }, - { title: '博客', docs: 'https://docs.rsshub.app/routes/bbs#wen-xue-cheng-bo-ke', source: '/myoverview/:id', target: '/wenxuecity/blog/:id' }, - ], - bbs: [ - { title: '最新主题', docs: 'https://docs.rsshub.app/routes/bbs#wen-xue-cheng-zui-xin-zhu-ti', source: '/:cat', target: '/wenxuecity/bbs/:cat' }, - { title: '最新主题 - 精华区', docs: 'https://docs.rsshub.app/routes/bbs#wen-xue-cheng-zui-xin-zhu-ti', source: '/:cat', target: '/wenxuecity/bbs/:cat/1' }, - { - title: '最热主题', - docs: 'https://docs.rsshub.app/routes/bbs#wen-xue-cheng-zui-re-zhu-ti', - source: '/?cid=*', - target: (params, url, document) => { - const cid = document && new URL(document.location).searchParams.get('cid'); - return `/wenxuecity/hot/${cid}`; - }, - }, - ], - }, - 'buaq.net': { _name: '不安全资讯', '.': [{ title: '不安全资讯', docs: 'http://docs.rsshub.app/routes/new-media#bu-an-quan', source: '/', target: '/buaq' }] }, - 'jian-ning.com': { _name: '建宁闲谈', '.': [{ title: '文章', docs: 'https://docs.rsshub.app/routes/blog#jian-ning-xian-tan', source: '/*', target: '/blogs/jianning' }] }, - 'matataki.io': { - _name: 'matataki', - www: [ - { title: '最热作品', docs: 'https://docs.rsshub.app/routes/new-media#matataki', source: '/article/', target: '/matataki/posts/hot' }, - { title: '最新作品', docs: 'https://docs.rsshub.app/routes/new-media#matataki', source: '/article/latest', target: '/matataki/posts/latest' }, - { title: '作者创作', docs: 'https://docs.rsshub.app/routes/new-media#matataki', source: '/user/:uid', target: (params) => `/matataki/users/${params.uid}/posts` }, - { title: 'Fan票关联作品', docs: 'https://docs.rsshub.app/routes/new-media#matataki', source: ['/token/:tokenId', '/token/:tokenId/circle'], target: (params) => `/matataki/tokens/${params.tokenId}/posts` }, - { - title: '标签关联作品', - docs: 'https://docs.rsshub.app/routes/new-media#matataki', - source: ['/tag/:tagId'], - target: (params, url) => { - const tagName = new URL(url).searchParams.get('name'); - return `/matataki/tags/${params.tagId}/${tagName}/posts`; - }, - }, - { title: '收藏夹', docs: 'https://docs.rsshub.app/routes/new-media#matataki', source: '/user/:uid/favlist/:fid', target: (params) => `/matataki/users/${params.uid}/favorites/${params.fid}/posts` }, - ], - }, - 'eventernote.com': { _name: 'Eventernote', www: [{ title: '声优活动及演唱会', docs: 'https://docs.rsshub.app/routes/anime#eventernote', source: '/actors/:name/:id/events', target: '/eventernote/actors/:name/:id' }] }, - 'huya.com': { _name: '虎牙直播', '.': [{ title: '直播间开播', docs: 'https://docs.rsshub.app/routes/live#hu-ya-zhi-bo-zhi-bo-jian-kai-bo', source: '/:id', target: '/huya/live/:id' }] }, - 'craigslist.org': { _name: 'Craigslist', '.': [{ title: '商品搜索列表', docs: 'https://docs.rsshub.app/routes/shopping#craigslist' }] }, - 'scboy.com': { - _name: 'scboy 论坛', - www: [ - { - title: '帖子', - docs: 'https://docs.rsshub.app/routes/bbs#scboy', - source: '', - target: (params, url) => { - const id = url.includes('thread') ? url.split('-')[1].split('.')[0] : ''; - return id ? `/scboy/thread/${id}` : ''; - }, - }, - ], - }, - 'cqut.edu.cn': { - _name: '重庆理工大学', - tz: [{ title: '通知', docs: 'https://docs.rsshub.app/routes/university#chong-qing-li-gong-da-xue', source: '/*' }], - lib: [{ title: '图书馆通知', docs: 'https://docs.rsshub.app/routes/university#chong-qing-li-gong-da-xue', source: '/*' }], - }, - 'cqwu.net': { - _name: '重庆文理学院', - www: [ - { - title: '通知', - docs: 'https://docs.rsshub.app/routes/university#chong-qing-wen-li-xue-yuan', - source: '/:type', - target: (params) => { - if (params.type === 'channel_7721.html') { - return '/cqwu/news/notify'; - } - }, - }, - { - title: '学术活动', - docs: 'https://docs.rsshub.app/routes/university#chong-qing-wen-li-xue-yuan', - source: '/:type', - target: (params) => { - if (params.type === 'channel_7722.html') { - return '/cqwu/news/academiceve'; - } - }, - }, - ], - }, - 'trakt.tv': { - _name: 'Trakt.tv', - '.': [ - { - title: '用户收藏', - docs: 'https://docs.rsshub.app/routes/multimedia#trakt-tv-yong-hu-shou-cang', - source: ['/users/:username/collection/:type/added', '/users/:username/collection'], - target: (params) => `/trakt/collection/${params.username}/${params.type || 'all'}`, - }, - ], - }, - 'furaffinity.net': { - _name: 'Fur Affinity', - www: [ - { title: '主页', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/', target: '/furaffinity/home' }, - { title: '浏览', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/browse/', target: '/furaffinity/browse' }, - { title: '站点状态', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/', target: '/furaffinity/status' }, - { - title: '搜索', - docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', - source: '/search/', - target: (params, url) => { - const keyword = new URL(url).searchParams.get('q'); - if (keyword) { - return `/furaffinity/search/${keyword}`; - } - }, - }, - { title: '用户主页简介', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/user/:username/', target: '/furaffinity/user/:username' }, - { title: '用户关注列表', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/watchlist/by/:username/', target: '/furaffinity/watching/:username' }, - { title: '用户被关注列表', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/watchlist/to/:username/', target: '/furaffinity/watchers/:username' }, - { title: '用户接受委托信息', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/commissions/:username/', target: '/furaffinity/commissions/:username' }, - { title: '用户的 Shouts 留言', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/user/:username/', target: '/furaffinity/shouts/:username' }, - { title: '用户的日记', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/journals/:username/', target: '/furaffinity/journals/:username' }, - { title: '用户的创作画廊', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/gallery/:username/', target: '/furaffinity/gallery/:username' }, - { title: '用户非正式作品', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/scraps/:username/', target: '/furaffinity/scraps/:username' }, - { title: '用户的喜爱列表', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/favorites/:username/', target: '/furaffinity/favorites/:username' }, - { title: '作品评论区', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/view/:id/', target: '/furaffinity/submission_comments/:id' }, - { title: '日记评论区', docs: 'https://docs.rsshub.app/routes/social-media#fur-affinity', source: '/journal/:id/', target: '/furaffinity/journal_comments/:id' }, - ], - }, - 'macwk.com': { _name: 'MacWk', '.': [{ title: '应用更新', docs: 'https://docs.rsshub.app/routes/program-update#macwk', source: '/soft/:name', target: '/macwk/soft/:name' }] }, - 'foreverblog.cn': { - _name: 'foreverblog', - www: [ - { - title: '十年之约', - docs: 'https://docs.rsshub.app/routes/social-media#foreverblog', - }, - ], - }, -}); diff --git a/babel.config.js b/babel.config.js new file mode 100644 index 00000000000000..a3092abe5333c5 --- /dev/null +++ b/babel.config.js @@ -0,0 +1,3 @@ +module.exports = { + presets: [['@babel/preset-env', { targets: { node: 'current' } }], '@babel/preset-typescript'], +}; diff --git a/docker-compose.yml b/docker-compose.yml index e404598334deaa..8b79ddf8085c28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: '3.9' - services: rsshub: # two ways to enable puppeteer: @@ -8,47 +6,45 @@ services: image: diygod/rsshub restart: always ports: - - '1200:1200' + - "1200:1200" environment: NODE_ENV: production CACHE_TYPE: redis - REDIS_URL: 'redis://redis:6379/' - PUPPETEER_WS_ENDPOINT: 'ws://browserless:3000' # marked - PROXY_URI: 'socks5h://warp-socks:9091' + REDIS_URL: "redis://redis:6379/" + PUPPETEER_WS_ENDPOINT: "ws://browserless:3000" # marked + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:1200/healthz"] + interval: 30s + timeout: 10s + retries: 3 depends_on: - redis - - browserless # marked + - browserless # marked - browserless: # marked - image: browserless/chrome # marked - restart: always # marked - ulimits: # marked - core: # marked - hard: 0 # marked - soft: 0 # marked + browserless: # marked + image: browserless/chrome # marked + restart: always # marked + ulimits: # marked + core: # marked + hard: 0 # marked + soft: 0 # marked + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:3000/pressure"] + interval: 30s + timeout: 10s + retries: 3 redis: image: redis:alpine restart: always volumes: - redis-data:/data - - warp-socks: - image: monius/docker-warp-socks:latest - privileged: true - volumes: - - /lib/modules:/lib/modules - cap_add: - - NET_ADMIN - - SYS_ADMIN - sysctls: - net.ipv6.conf.all.disable_ipv6: 0 - net.ipv4.conf.all.src_valid_mark: 1 healthcheck: - test: ["CMD", "curl", "-f", "https://www.cloudflare.com/cdn-cgi/trace"] + test: ["CMD", "redis-cli", "ping"] interval: 30s timeout: 10s retries: 5 + start_period: 5s volumes: redis-data: diff --git a/eslint.config.mjs b/eslint.config.mjs new file mode 100644 index 00000000000000..b7137de1118c2d --- /dev/null +++ b/eslint.config.mjs @@ -0,0 +1,276 @@ +import prettier from 'eslint-plugin-prettier'; +import stylistic from '@stylistic/eslint-plugin'; +import unicorn from 'eslint-plugin-unicorn'; +import typescriptEslint from '@typescript-eslint/eslint-plugin'; +import globals from 'globals'; +import tsParser from '@typescript-eslint/parser'; +import parser from 'yaml-eslint-parser'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import js from '@eslint/js'; +import { FlatCompat } from '@eslint/eslintrc'; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const compat = new FlatCompat({ + baseDirectory: __dirname, + recommendedConfig: js.configs.recommended, + allConfig: js.configs.all +}); + +export default [{ + ignores: [ + '**/coverage', + '**/.vscode', + '**/docker-compose.yml', + '!.github', + 'assets/build/radar-rules.js', + 'lib/routes-deprecated', + 'lib/router.js', + '**/babel.config.js', + 'scripts/docker/minify-docker.js', + ], +}, ...compat.extends( + 'eslint:recommended', + 'plugin:n/recommended', + 'plugin:unicorn/recommended', + 'plugin:prettier/recommended', + 'plugin:yml/recommended', + 'plugin:@typescript-eslint/recommended', +), { + plugins: { + prettier, + '@stylistic': stylistic, + unicorn, + '@typescript-eslint': typescriptEslint, + }, + + languageOptions: { + globals: { + ...globals.node, + ...globals.browser, + }, + + parser: tsParser, + ecmaVersion: 'latest', + sourceType: 'module', + }, + + rules: { + // possible problems + 'array-callback-return': ['error', { + allowImplicit: true, + }], + + 'no-await-in-loop': 'error', + 'no-control-regex': 'off', + 'no-duplicate-imports': 'error', + 'no-prototype-builtins': 'off', + + // suggestions + 'arrow-body-style': 'error', + 'block-scoped-var': 'error', + curly: 'error', + 'dot-notation': 'error', + eqeqeq: 'error', + + 'default-case': ['warn', { + commentPattern: '^no default$', + }], + + 'default-case-last': 'error', + 'no-console': 'error', + 'no-eval': 'error', + 'no-extend-native': 'error', + 'no-extra-label': 'error', + + 'no-implicit-coercion': ['error', { + boolean: false, + number: false, + string: false, + disallowTemplateShorthand: true, + }], + + 'no-implicit-globals': 'error', + 'no-labels': 'error', + 'no-multi-str': 'error', + 'no-new-func': 'error', + 'no-restricted-imports': 'error', + + 'no-restricted-syntax': ['warn', { + selector: "CallExpression[callee.property.name='get'][arguments.length=0]", + message: "Please use .toArray() instead.", + }, { + selector: "CallExpression[callee.property.name='toArray'] MemberExpression[object.callee.property.name='map']", + message: "Please use .toArray() before .map().", + }], + + 'no-unneeded-ternary': 'error', + 'no-useless-computed-key': 'error', + 'no-useless-concat': 'warn', + 'no-useless-rename': 'error', + 'no-var': 'error', + 'object-shorthand': 'error', + 'prefer-arrow-callback': 'error', + 'prefer-const': 'error', + 'prefer-object-has-own': 'error', + 'no-useless-escape': 'warn', + + 'prefer-regex-literals': ['error', { + disallowRedundantWrapping: true, + }], + + 'require-await': 'error', + + // typescript + '@typescript-eslint/ban-ts-comment': 'off', + '@typescript-eslint/no-explicit-any': 'off', + '@typescript-eslint/no-var-requires': 'off', + + '@typescript-eslint/no-unused-expressions': ['error', { + allowShortCircuit: true, + allowTernary: true, + }], + + // unicorn + 'unicorn/consistent-destructuring': 'warn', + 'unicorn/consistent-function-scoping': 'warn', + 'unicorn/explicit-length-check': 'off', + + 'unicorn/filename-case': ['error', { + case: 'kebabCase', + ignore: [String.raw`.*\.(yaml|yml)$`, String.raw`RequestInProgress\.js$`], + }], + + 'unicorn/new-for-builtins': 'off', + 'unicorn/no-array-callback-reference': 'warn', + 'unicorn/no-array-reduce': 'warn', + 'unicorn/no-await-expression-member': 'off', + 'unicorn/no-empty-file': 'warn', + 'unicorn/no-hex-escape': 'warn', + 'unicorn/no-null': 'off', + 'unicorn/no-object-as-default-parameter': 'warn', + 'unicorn/no-process-exit': 'off', + 'unicorn/no-useless-switch-case': 'off', + + 'unicorn/no-useless-undefined': ['error', { + checkArguments: false, + }], + + 'unicorn/numeric-separators-style': ['warn', { + onlyIfContainsSeparator: false, + + number: { + minimumDigits: 7, + groupLength: 3, + }, + + binary: { + minimumDigits: 9, + groupLength: 4, + }, + + octal: { + minimumDigits: 9, + groupLength: 4, + }, + + hexadecimal: { + minimumDigits: 5, + groupLength: 2, + }, + }], + + 'unicorn/prefer-code-point': 'warn', + 'unicorn/prefer-global-this': 'off', + 'unicorn/prefer-logical-operator-over-ternary': 'warn', + 'unicorn/prefer-module': 'off', + 'unicorn/prefer-node-protocol': 'off', + + 'unicorn/prefer-number-properties': ['warn', { + checkInfinity: false, + }], + + 'unicorn/prefer-object-from-entries': 'warn', + 'unicorn/prefer-regexp-test': 'warn', + 'unicorn/prefer-spread': 'warn', + 'unicorn/prefer-string-replace-all': 'warn', + 'unicorn/prefer-string-slice': 'off', + + 'unicorn/prefer-switch': ['warn', { + emptyDefaultCase: 'do-nothing-comment', + }], + + 'unicorn/prefer-top-level-await': 'off', + 'unicorn/prevent-abbreviations': 'off', + 'unicorn/switch-case-braces': ['error', 'avoid'], + 'unicorn/text-encoding-identifier-case': 'off', + + // formatting rules + '@stylistic/arrow-parens': 'error', + '@stylistic/arrow-spacing': 'error', + '@stylistic/comma-spacing': 'error', + '@stylistic/comma-style': 'error', + '@stylistic/function-call-spacing': 'error', + '@stylistic/keyword-spacing': 'error', + '@stylistic/linebreak-style': 'error', + + '@stylistic/lines-around-comment': ['error', { + beforeBlockComment: false, + }], + + '@stylistic/no-multiple-empty-lines': 'error', + '@stylistic/no-trailing-spaces': 'error', + '@stylistic/rest-spread-spacing': 'error', + '@stylistic/semi': 'error', + '@stylistic/space-before-blocks': 'error', + '@stylistic/space-in-parens': 'error', + '@stylistic/space-infix-ops': 'error', + '@stylistic/space-unary-ops': 'error', + '@stylistic/spaced-comment': 'error', + + // https://github.com/eslint-community/eslint-plugin-n + // node specific rules + 'n/no-extraneous-require': ['error', { + allowModules: [ + 'puppeteer-extra-plugin-user-preferences', + 'puppeteer-extra-plugin-user-data-dir', + ], + }], + + 'n/no-deprecated-api': 'warn', + 'n/no-missing-import': 'off', + 'n/no-missing-require': 'off', + 'n/no-process-exit': 'off', + 'n/no-unpublished-import': 'off', + + 'n/no-unpublished-require': ['error', { + allowModules: ['tosource'], + }], + + 'prettier/prettier': 'off', + + 'yml/quotes': ['error', { + prefer: 'single', + }], + + 'yml/no-empty-mapping-value': 'off', + }, +}, { + files: ['.puppeteerrc.cjs', 'api/vercel.ts'], + rules: { + '@typescript-eslint/no-require-imports': 'off', + } +}, { + files: ['**/*.yaml', '**/*.yml'], + + languageOptions: { + parser, + }, + + rules: { + 'lines-around-comment': ['error', { + beforeBlockComment: false, + }], + }, +}]; diff --git a/jsconfig.json b/jsconfig.json deleted file mode 100644 index 716d74dedff67e..00000000000000 --- a/jsconfig.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "compilerOptions": { - "baseUrl": ".", - "paths": { - "@/*": ["./lib/*"] - } - }, - "include": ["./lib/**/*"] -} diff --git a/lib/api/category/one.ts b/lib/api/category/one.ts new file mode 100644 index 00000000000000..06cadeb6504bf4 --- /dev/null +++ b/lib/api/category/one.ts @@ -0,0 +1,81 @@ +import { namespaces } from '@/registry'; +import { z, createRoute, RouteHandler } from '@hono/zod-openapi'; + +const categoryList: Record = {}; + +for (const namespace in namespaces) { + for (const path in namespaces[namespace].routes) { + if (namespaces[namespace].routes[path].categories?.length) { + for (const category of namespaces[namespace].routes[path].categories!) { + if (!categoryList[category]) { + categoryList[category] = {}; + } + if (!categoryList[category][namespace]) { + categoryList[category][namespace] = { + ...namespaces[namespace], + routes: {}, + }; + } + categoryList[category][namespace].routes[path] = namespaces[namespace].routes[path]; + } + } + } +} + +const ParamsSchema = z.object({ + category: z.string().openapi({ + param: { + name: 'category', + in: 'path', + }, + example: 'popular', + }), +}); + +const QuerySchema = z.object({ + categories: z + .string() + .transform((val) => val.split(',')) + .optional(), + lang: z.string().optional(), +}); + +const route = createRoute({ + method: 'get', + path: '/category/{category}', + tags: ['Category'], + request: { + query: QuerySchema, + params: ParamsSchema, + }, + responses: { + 200: { + description: 'Namespace list by categories and language', + }, + }, +}); + +const handler: RouteHandler = (ctx) => { + const { categories, lang } = ctx.req.valid('query'); + const { category } = ctx.req.valid('param'); + + let allCategories = [category]; + if (categories && categories.length > 0) { + allCategories = [...allCategories, ...categories]; + } + + // Get namespaces that exist in all requested categories + const commonNamespaces = Object.keys(categoryList[category] || {}).filter((namespace) => allCategories.every((cat) => categoryList[cat]?.[namespace])); + + // Create result directly from common namespaces + let result = Object.fromEntries(commonNamespaces.map((namespace) => [namespace, categoryList[category][namespace]])); + + // Filter by language if provided + if (lang) { + result = Object.fromEntries(Object.entries(result).filter(([, value]) => value.lang === lang)); + } + + return ctx.json(result); +}; + +export { route, handler }; diff --git a/lib/api/follow/config.ts b/lib/api/follow/config.ts new file mode 100644 index 00000000000000..6465470dd845be --- /dev/null +++ b/lib/api/follow/config.ts @@ -0,0 +1,23 @@ +import { config } from '@/config'; +import { createRoute, RouteHandler } from '@hono/zod-openapi'; + +const route = createRoute({ + method: 'get', + path: '/follow/config', + tags: ['Follow'], + responses: { + 200: { + description: 'Follow config', + }, + }, +}); + +const handler: RouteHandler = (ctx) => + ctx.json({ + ownerUserId: config.follow.ownerUserId, + description: config.follow.description, + price: config.follow.price, + userLimit: config.follow.userLimit, + }); + +export { route, handler }; diff --git a/lib/api/index.ts b/lib/api/index.ts new file mode 100644 index 00000000000000..064852019b7244 --- /dev/null +++ b/lib/api/index.ts @@ -0,0 +1,39 @@ +// import { route as rulesRoute, handler as rulesHandler } from '@/api/radar/rules'; +import { route as namespaceAllRoute, handler as namespaceAllHandler } from '@/api/namespace/all'; +import { route as namespaceOneRoute, handler as namespaceOneHandler } from '@/api/namespace/one'; +import { route as radarRulesAllRoute, handler as radarRulesAllHandler } from '@/api/radar/rules/all'; +import { route as radarRulesOneRoute, handler as radarRulesOneHandler } from '@/api/radar/rules/one'; +import { route as categoryOneRoute, handler as categoryOneHandler } from '@/api/category/one'; +import { route as followConfigRoute, handler as followConfigHandler } from '@/api/follow/config'; +import { OpenAPIHono } from '@hono/zod-openapi'; +import { apiReference } from '@scalar/hono-api-reference'; + +const app = new OpenAPIHono(); + +app.openapi(namespaceAllRoute, namespaceAllHandler); +app.openapi(namespaceOneRoute, namespaceOneHandler); +app.openapi(radarRulesAllRoute, radarRulesAllHandler); +app.openapi(radarRulesOneRoute, radarRulesOneHandler); +app.openapi(categoryOneRoute, categoryOneHandler); +app.openapi(followConfigRoute, followConfigHandler); + +const docs = app.getOpenAPI31Document({ + openapi: '3.1.0', + info: { + version: '0.0.1', + title: 'RSSHub API', + }, +}); +for (const path in docs.paths) { + docs.paths[`/api${path}`] = docs.paths[path]; + delete docs.paths[path]; +} +app.get('/openapi.json', (ctx) => ctx.json(docs)); +app.get( + '/reference', + apiReference({ + spec: { content: docs }, + }) +); + +export default app; diff --git a/lib/api/namespace/all.ts b/lib/api/namespace/all.ts new file mode 100644 index 00000000000000..768d2a0c9b196a --- /dev/null +++ b/lib/api/namespace/all.ts @@ -0,0 +1,17 @@ +import { namespaces } from '@/registry'; +import { createRoute, RouteHandler } from '@hono/zod-openapi'; + +const route = createRoute({ + method: 'get', + path: '/namespace', + tags: ['Namespace'], + responses: { + 200: { + description: 'Information about all namespaces', + }, + }, +}); + +const handler: RouteHandler = (ctx) => ctx.json(namespaces); + +export { route, handler }; diff --git a/lib/api/namespace/one.ts b/lib/api/namespace/one.ts new file mode 100644 index 00000000000000..cd09375ce8131d --- /dev/null +++ b/lib/api/namespace/one.ts @@ -0,0 +1,33 @@ +import { namespaces } from '@/registry'; +import { z, createRoute, RouteHandler } from '@hono/zod-openapi'; + +const ParamsSchema = z.object({ + namespace: z.string().openapi({ + param: { + name: 'namespace', + in: 'path', + }, + example: 'github', + }), +}); + +const route = createRoute({ + method: 'get', + path: '/namespace/{namespace}', + tags: ['Namespace'], + request: { + params: ParamsSchema, + }, + responses: { + 200: { + description: 'Information about a namespace', + }, + }, +}); + +const handler: RouteHandler = (ctx) => { + const { namespace } = ctx.req.valid('param'); + return ctx.json(namespaces[namespace]); +}; + +export { route, handler }; diff --git a/lib/api/radar/rules/all.ts b/lib/api/radar/rules/all.ts new file mode 100644 index 00000000000000..7266a8291043b3 --- /dev/null +++ b/lib/api/radar/rules/all.ts @@ -0,0 +1,56 @@ +import { namespaces } from '@/registry'; +import { parse } from 'tldts'; +import { RadarDomain } from '@/types'; +import { createRoute, RouteHandler } from '@hono/zod-openapi'; + +const radar: { + [domain: string]: RadarDomain; +} = {}; + +for (const namespace in namespaces) { + for (const path in namespaces[namespace].routes) { + const realPath = `/${namespace}${path}`; + const data = namespaces[namespace].routes[path]; + if (data.radar?.length) { + for (const radarItem of data.radar) { + const parsedDomain = parse(new URL('https://' + radarItem.source[0]).hostname); + const subdomain = parsedDomain.subdomain || '.'; + const domain = parsedDomain.domain; + if (domain) { + if (!radar[domain]) { + radar[domain] = { + _name: namespaces[namespace].name, + } as RadarDomain; + } + if (!radar[domain][subdomain]) { + radar[domain][subdomain] = []; + } + radar[domain][subdomain].push({ + title: radarItem.title || data.name, + docs: `https://docs.rsshub.app/routes/${data.categories?.[0] || 'other'}`, + source: radarItem.source.map((source) => { + const sourceURL = new URL('https://' + source); + return sourceURL.pathname + sourceURL.search + sourceURL.hash; + }), + target: radarItem.target ? `/${namespace}${radarItem.target}` : realPath, + }); + } + } + } + } +} + +const route = createRoute({ + method: 'get', + path: '/radar/rules', + tags: ['Radar'], + responses: { + 200: { + description: 'All Radar rules', + }, + }, +}); + +const handler: RouteHandler = (ctx) => ctx.json(radar); + +export { route, handler }; diff --git a/lib/api/radar/rules/one.ts b/lib/api/radar/rules/one.ts new file mode 100644 index 00000000000000..6cf859875bc3d8 --- /dev/null +++ b/lib/api/radar/rules/one.ts @@ -0,0 +1,72 @@ +import { namespaces } from '@/registry'; +import { parse } from 'tldts'; +import { RadarDomain } from '@/types'; +import { z, createRoute, RouteHandler } from '@hono/zod-openapi'; + +const radar: { + [domain: string]: RadarDomain; +} = {}; + +for (const namespace in namespaces) { + for (const path in namespaces[namespace].routes) { + const realPath = `/${namespace}${path}`; + const data = namespaces[namespace].routes[path]; + if (data.radar?.length) { + for (const radarItem of data.radar) { + const parsedDomain = parse(new URL('https://' + radarItem.source[0]).hostname); + const subdomain = parsedDomain.subdomain || '.'; + const domain = parsedDomain.domain; + if (domain) { + if (!radar[domain]) { + radar[domain] = { + _name: namespaces[namespace].name, + } as RadarDomain; + } + if (!radar[domain][subdomain]) { + radar[domain][subdomain] = []; + } + radar[domain][subdomain].push({ + title: radarItem.title || data.name, + docs: `https://docs.rsshub.app/routes/${data.categories?.[0] || 'other'}`, + source: radarItem.source.map((source) => { + const sourceURL = new URL('https://' + source); + return sourceURL.pathname + sourceURL.search + sourceURL.hash; + }), + target: radarItem.target ? `/${namespace}${radarItem.target}` : realPath, + }); + } + } + } + } +} + +const ParamsSchema = z.object({ + domain: z.string().openapi({ + param: { + name: 'domain', + in: 'path', + }, + example: 'github.com', + }), +}); + +const route = createRoute({ + method: 'get', + path: '/radar/rules/{domain}', + tags: ['Radar'], + request: { + params: ParamsSchema, + }, + responses: { + 200: { + description: 'Radar rules for a domain name (does not support subdomains)', + }, + }, +}); + +const handler: RouteHandler = (ctx) => { + const { domain } = ctx.req.valid('param'); + return ctx.json(radar[domain]); +}; + +export { route, handler }; diff --git a/lib/api_router.js b/lib/api_router.js deleted file mode 100644 index b452d60666babb..00000000000000 --- a/lib/api_router.js +++ /dev/null @@ -1,26 +0,0 @@ -const Router = require('@koa/router'); -const router = new Router(); - -router.get('/routes/:name?', (ctx) => { - const result = {}; - let counter = 0; - - const maintainer = require('./maintainer'); - Object.keys(maintainer).forEach((i) => { - const path = i; - const top = path.split('/')[1]; - - if (!ctx.params.name || top === ctx.params.name) { - if (result[top]) { - result[top].routes.push(path); - } else { - result[top] = { routes: [path] }; - } - counter++; - } - }); - - ctx.body = { counter, result }; -}); - -module.exports = router; diff --git a/lib/app.js b/lib/app.js deleted file mode 100644 index 39b91f574e1c78..00000000000000 --- a/lib/app.js +++ /dev/null @@ -1,94 +0,0 @@ -const moduleAlias = require('module-alias'); -moduleAlias.addAlias('@', () => __dirname); - -require('./utils/request-wrapper'); - -const Koa = require('koa'); -const logger = require('./utils/logger'); - -const onerror = require('./middleware/onerror'); -const header = require('./middleware/header'); -const utf8 = require('./middleware/utf8'); -const cache = require('./middleware/cache'); -const parameter = require('./middleware/parameter'); -const template = require('./middleware/template'); -const favicon = require('koa-favicon'); -const serve = require('koa-static'); -const debug = require('./middleware/debug'); -const accessControl = require('./middleware/access-control'); -const antiHotlink = require('./middleware/anti-hotlink'); -const loadOnDemand = require('./middleware/load-on-demand'); - -const router = require('./router'); -const core_router = require('./core_router'); -const protected_router = require('./protected_router'); -const mount = require('koa-mount'); - -// API related -const apiTemplate = require('./middleware/api-template'); -const api_router = require('./api_router'); -const apiResponseHandler = require('./middleware/api-response-handler'); - -process.on('uncaughtException', (e) => { - logger.error('uncaughtException: ' + e); -}); - -const app = new Koa(); -app.proxy = true; - -// favicon -app.use(favicon(__dirname + '/favicon.png', { maxAge: 31536000000 })); -app.use(serve(__dirname + '/static', { maxage: 31536000000 })); - -// global error handing -app.use(onerror); - -app.use(accessControl); - -// 7 debug -app.context.debug = { - hitCache: 0, - request: 0, - etag: 0, - paths: [], - routes: [], - errorPaths: [], - errorRoutes: [], -}; -app.use(debug); - -// 6 set header -app.use(header); - -// 5 fix incorrect `utf-8` characters -app.use(utf8); - -app.use(apiTemplate); -app.use(apiResponseHandler()); - -// 4 generate body -app.use(template); -// anti-hotlink -app.use(antiHotlink); - -// 3 filter content -app.use(parameter); - -// No Cache routes -app.use(mount('/', core_router.routes())).use(core_router.allowedMethods()); -// API router -app.use(mount('/api', api_router.routes())).use(api_router.allowedMethods()); - -// 2 cache -app.use(cache(app)); - -// 1 load on demand -app.use(loadOnDemand(app)); - -// router -app.use(mount('/', router.routes())).use(router.allowedMethods()); - -// routes the require authentication -app.use(mount('/protected', protected_router.routes())).use(protected_router.allowedMethods()); - -module.exports = app; diff --git a/lib/app.test.ts b/lib/app.test.ts new file mode 100644 index 00000000000000..b81d62ee55382e --- /dev/null +++ b/lib/app.test.ts @@ -0,0 +1,11 @@ +import { describe, expect, it } from 'vitest'; + +import app from '@/app'; + +describe('index', () => { + it('serve index', async () => { + const res = await app.request('/'); + expect(res.status).toBe(200); + expect(await res.text()).toContain('Welcome to RSSHub!'); + }); +}); diff --git a/lib/app.tsx b/lib/app.tsx new file mode 100644 index 00000000000000..2747dcfd27a3c3 --- /dev/null +++ b/lib/app.tsx @@ -0,0 +1,57 @@ +import '@/utils/request-rewriter'; + +import { Hono } from 'hono'; + +import { compress } from 'hono/compress'; +import mLogger from '@/middleware/logger'; +import cache from '@/middleware/cache'; +import template from '@/middleware/template'; +import sentry from '@/middleware/sentry'; +import accessControl from '@/middleware/access-control'; +import debug from '@/middleware/debug'; +import header from '@/middleware/header'; +import antiHotlink from '@/middleware/anti-hotlink'; +import parameter from '@/middleware/parameter'; +import trace from '@/middleware/trace'; +import { jsxRenderer } from 'hono/jsx-renderer'; +import { trimTrailingSlash } from 'hono/trailing-slash'; + +import logger from '@/utils/logger'; + +import { notFoundHandler, errorHandler } from '@/errors'; +import registry from '@/registry'; +import api from '@/api'; + +process.on('uncaughtException', (e) => { + logger.error('uncaughtException: ' + e); +}); + +const app = new Hono(); + +app.use(trimTrailingSlash()); +app.use(compress()); + +app.use( + jsxRenderer(({ children }) => <>{children}, { + docType: '', + stream: {}, + }) +); +app.use(mLogger); +app.use(trace); +app.use(sentry); +app.use(accessControl); +app.use(debug); +app.use(template); +app.use(header); +app.use(antiHotlink); +app.use(parameter); +app.use(cache); + +app.route('/', registry); +app.route('/api', api); + +app.notFound(notFoundHandler); +app.onError(errorHandler); + +export default app; diff --git a/lib/favicon.png b/lib/assets/favicon.png similarity index 100% rename from lib/favicon.png rename to lib/assets/favicon.png diff --git a/lib/assets/logo.png b/lib/assets/logo.png new file mode 100644 index 00000000000000..df91fde1f6cff1 Binary files /dev/null and b/lib/assets/logo.png differ diff --git a/lib/config.js b/lib/config.js deleted file mode 100644 index d435c4e50e0dd4..00000000000000 --- a/lib/config.js +++ /dev/null @@ -1,338 +0,0 @@ -require('dotenv').config(); -const randUserAgent = require('./utils/rand-user-agent'); -let envs = process.env; -let value; -const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)'; - -const calculateValue = () => { - const bilibili_cookies = {}; - const twitter_tokens = {}; - const email_config = {}; - const discuz_cookies = {}; - const medium_cookies = {}; - const discourse_config = {}; - - for (const name in envs) { - if (name.startsWith('BILIBILI_COOKIE_')) { - const uid = name.slice(16); - bilibili_cookies[uid] = envs[name]; - } else if (name.startsWith('TWITTER_TOKEN_')) { - const id = name.slice(14); - twitter_tokens[id] = envs[name]; - } else if (name.startsWith('EMAIL_CONFIG_')) { - const id = name.slice(13); - email_config[id] = envs[name]; - } else if (name.startsWith('DISCUZ_COOKIE_')) { - const cid = name.slice(14); - discuz_cookies[cid] = envs[name]; - } else if (name.startsWith('MEDIUM_COOKIE_')) { - const username = name.slice(14).toLowerCase(); - medium_cookies[username] = envs[name]; - } else if (name.startsWith('DISCOURSE_CONFIG_')) { - const id = name.slice('DISCOURSE_CONFIG_'.length); - discourse_config[id] = JSON.parse(envs[name]); - } - } - - value = { - // app config - disallowRobot: envs.DISALLOW_ROBOT !== '0' && envs.DISALLOW_ROBOT !== 'false', - enableCluster: envs.ENABLE_CLUSTER, - isPackage: envs.IS_PACKAGE, - nodeName: envs.NODE_NAME, - puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT, - chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH, - // network - connect: { - port: envs.PORT || 1200, // 监听端口 - socket: envs.SOCKET || null, // 监听 Unix Socket, null 为禁用 - }, - listenInaddrAny: envs.LISTEN_INADDR_ANY || 1, // 是否允许公网连接，取值 0 1 - requestRetry: parseInt(envs.REQUEST_RETRY) || 2, // 请求失败重试次数 - requestTimeout: parseInt(envs.REQUEST_TIMEOUT) || 30000, // Milliseconds to wait for the server to end the response before aborting the request - ua: envs.UA ? envs.UA : envs.NO_RANDOM_UA === 'true' || envs.NO_RANDOM_UA === '1' ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' }), - trueUA: TRUE_UA, - // cors request - allowOrigin: envs.ALLOW_ORIGIN, - // cache - cache: { - type: typeof envs.CACHE_TYPE === 'undefined' ? 'memory' : envs.CACHE_TYPE, // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存 - requestTimeout: parseInt(envs.CACHE_REQUEST_TIMEOUT) || 60, - routeExpire: parseInt(envs.CACHE_EXPIRE) || 5 * 60, // 路由缓存时间，单位为秒 - contentExpire: parseInt(envs.CACHE_CONTENT_EXPIRE) || 1 * 60 * 60, // 不变内容缓存时间，单位为秒 - }, - memory: { - max: parseInt(envs.MEMORY_MAX) || Math.pow(2, 8), // The maximum number of items that remain in the cache. This must be a positive finite intger. - // https://github.com/isaacs/node-lru-cache#options - }, - redis: { - url: envs.REDIS_URL || 'redis://localhost:6379/', - }, - // proxy - proxyUri: envs.PROXY_URI, - proxy: { - protocol: envs.PROXY_PROTOCOL, - host: envs.PROXY_HOST, - port: envs.PROXY_PORT, - auth: envs.PROXY_AUTH, - url_regex: envs.PROXY_URL_REGEX || '.*', - }, - proxyStrategy: envs.PROXY_STRATEGY || 'all', // all / on_retry - reverseProxyUrl: envs.REVERSE_PROXY_URL, - // auth - authentication: { - name: envs.HTTP_BASIC_AUTH_NAME || 'usernam3', - pass: envs.HTTP_BASIC_AUTH_PASS || 'passw0rd', - }, - // access control - blacklist: envs.BLACKLIST && envs.BLACKLIST.split(','), - whitelist: envs.WHITELIST && envs.WHITELIST.split(','), - allowLocalhost: envs.ALLOW_LOCALHOST, - accessKey: envs.ACCESS_KEY, - // logging - // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示 - debugInfo: envs.DEBUG_INFO || 'true', - loggerLevel: envs.LOGGER_LEVEL || 'info', - noLogfiles: envs.NO_LOGFILES, - showLoggerTimestamp: envs.SHOW_LOGGER_TIMESTAMP, - sentry: { - dsn: envs.SENTRY, - routeTimeout: parseInt(envs.SENTRY_ROUTE_TIMEOUT) || 30000, - }, - // feed config - hotlink: { - template: envs.HOTLINK_TEMPLATE, - includePaths: envs.HOTLINK_INCLUDE_PATHS && envs.HOTLINK_INCLUDE_PATHS.split(','), - excludePaths: envs.HOTLINK_EXCLUDE_PATHS && envs.HOTLINK_EXCLUDE_PATHS.split(','), - }, - feature: { - allow_user_hotlink_template: envs.ALLOW_USER_HOTLINK_TEMPLATE === 'true', - filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', - allow_user_supply_unsafe_domain: envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN === 'true', - }, - suffix: envs.SUFFIX, - titleLengthLimit: parseInt(envs.TITLE_LENGTH_LIMIT) || 150, - - // Route-specific Configurations - bilibili: { - cookies: bilibili_cookies, - }, - bitbucket: { - username: envs.BITBUCKET_USERNAME, - password: envs.BITBUCKET_PASSWORD, - }, - btbyr: { - host: envs.BTBYR_HOST, - cookies: envs.BTBYR_COOKIE, - }, - bupt: { - portal_cookie: envs.BUPT_PORTAL_COOKIE, - }, - chuiniu: { - member: envs.CHUINIU_MEMBER, - }, - civitai: { - cookie: envs.CIVITAI_COOKIE, - }, - dida365: { - username: envs.DIDA365_USERNAME, - password: envs.DIDA365_PASSWORD, - }, - discord: { - authorization: envs.DISCORD_AUTHORIZATION, - }, - discourse: { - config: discourse_config, - }, - discuz: { - cookies: discuz_cookies, - }, - disqus: { - api_key: envs.DISQUS_API_KEY, - }, - douban: { - cookie: envs.DOUBAN_COOKIE, - }, - ehentai: { - ipb_member_id: envs.EH_IPB_MEMBER_ID, - ipb_pass_hash: envs.EH_IPB_PASS_HASH, - sk: envs.EH_SK, - igneous: envs.EH_IGNEOUS, - star: envs.EH_STAR, - img_proxy: envs.EH_IMG_PROXY, - }, - email: { - config: email_config, - }, - fanbox: { - session: envs.FANBOX_SESSION_ID, - }, - fanfou: { - consumer_key: envs.FANFOU_CONSUMER_KEY, - consumer_secret: envs.FANFOU_CONSUMER_SECRET, - username: envs.FANFOU_USERNAME, - password: envs.FANFOU_PASSWORD, - }, - fantia: { - cookies: envs.FANTIA_COOKIE, - }, - game4399: { - cookie: envs.GAME_4399, - }, - github: { - access_token: envs.GITHUB_ACCESS_TOKEN, - }, - gitee: { - access_token: envs.GITEE_ACCESS_TOKEN, - }, - google: { - fontsApiKey: envs.GOOGLE_FONTS_API_KEY, - }, - hefeng: { - // weather - key: envs.HEFENG_KEY, - }, - infzm: { - cookie: envs.INFZM_COOKIE, - }, - initium: { - username: envs.INITIUM_USERNAME, - password: envs.INITIUM_PASSWORD, - bearertoken: envs.INITIUM_BEARER_TOKEN, - iap_receipt: envs.INITIUM_IAP_RECEIPT, - }, - instagram: { - username: envs.IG_USERNAME, - password: envs.IG_PASSWORD, - proxy: envs.IG_PROXY, - cookie: envs.IG_COOKIE, - }, - iwara: { - username: envs.IWARA_USERNAME, - password: envs.IWARA_PASSWORD, - }, - lastfm: { - api_key: envs.LASTFM_API_KEY, - }, - manhuagui: { - cookie: envs.MHGUI_COOKIE, - }, - mastodon: { - apiHost: envs.MASTODON_API_HOST, - accessToken: envs.MASTODON_API_ACCESS_TOKEN, - acctDomain: envs.MASTODON_API_ACCT_DOMAIN, - }, - medium: { - cookies: medium_cookies, - articleCookie: envs.MEDIUM_ARTICLE_COOKIE || '', - }, - miniflux: { - instance: envs.MINIFLUX_INSTANCE || 'https://reader.miniflux.app', - token: envs.MINIFLUX_TOKEN || '', - }, - ncm: { - cookies: envs.NCM_COOKIES || '', - }, - newrank: { - cookie: envs.NEWRANK_COOKIE, - }, - nga: { - uid: envs.NGA_PASSPORT_UID, - cid: envs.NGA_PASSPORT_CID, - }, - nhentai: { - username: envs.NHENTAI_USERNAME, - password: envs.NHENTAI_PASSWORD, - }, - notion: { - key: envs.NOTION_TOKEN, - }, - pianyuan: { - cookie: envs.PIANYUAN_COOKIE, - }, - pixabay: { - key: envs.PIXABAY_KEY, - }, - pixiv: { - refreshToken: envs.PIXIV_REFRESHTOKEN, - bypassCdn: envs.PIXIV_BYPASS_CDN && envs.PIXIV_BYPASS_CDN !== '0' && envs.PIXIV_BYPASS_CDN !== 'false', - bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net', - bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query', - imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re', - }, - pkubbs: { - cookie: envs.PKUBBS_COOKIE, - }, - saraba1st: { - cookie: envs.SARABA1ST_COOKIE, - }, - sehuatang: { - cookie: envs.SEHUATANG_COOKIE, - }, - scboy: { - token: envs.SCBOY_BBS_TOKEN, - }, - scihub: { - host: envs.SCIHUB_HOST || 'https://sci-hub.se/', - }, - spotify: { - clientId: envs.SPOTIFY_CLIENT_ID, - clientSecret: envs.SPOTIFY_CLIENT_SECRET, - refreshToken: envs.SPOTIFY_REFRESHTOKEN, - }, - telegram: { - token: envs.TELEGRAM_TOKEN, - }, - tophub: { - cookie: envs.TOPHUB_COOKIE, - }, - twitter: { - consumer_key: envs.TWITTER_CONSUMER_KEY, - consumer_secret: envs.TWITTER_CONSUMER_SECRET, - tokens: twitter_tokens, - authorization: envs.TWITTER_WEBAPI_AUTHORIZAION && envs.TWITTER_WEBAPI_AUTHORIZAION.split(','), - }, - weibo: { - app_key: envs.WEIBO_APP_KEY, - app_secret: envs.WEIBO_APP_SECRET, - cookies: envs.WEIBO_COOKIES, - redirect_url: envs.WEIBO_REDIRECT_URL, - }, - wenku8: { - cookie: envs.WENKU8_COOKIE, - }, - wordpress: { - cdnUrl: envs.WORDPRESS_CDN, - }, - xiaoyuzhou: { - device_id: envs.XIAOYUZHOU_ID, - refresh_token: envs.XIAOYUZHOU_TOKEN, - }, - ximalaya: { - token: envs.XIMALAYA_TOKEN, - }, - youtube: { - key: envs.YOUTUBE_KEY, - clientId: envs.YOUTUBE_CLIENT_ID, - clientSecret: envs.YOUTUBE_CLIENT_SECRET, - refreshToken: envs.YOUTUBE_REFRESH_TOKEN, - }, - zhihu: { - cookies: envs.ZHIHU_COOKIES, - }, - zodgame: { - cookie: envs.ZODGAME_COOKIE, - }, - }; -}; -calculateValue(); - -module.exports = { - set: (env) => { - envs = Object.assign(process.env, env); - calculateValue(); - }, - get value() { - return value; - }, -}; diff --git a/lib/config.test.ts b/lib/config.test.ts new file mode 100644 index 00000000000000..edf38055a6dfcc --- /dev/null +++ b/lib/config.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it, afterEach, vi } from 'vitest'; + +afterEach(() => { + vi.resetModules(); +}); + +describe('config', () => { + it('bilibilib cookie', async () => { + process.env.BILIBILI_COOKIE_12 = 'cookie1'; + process.env.BILIBILI_COOKIE_34 = 'cookie2'; + + const { config } = await import('./config'); + expect(config.bilibili.cookies).toMatchObject({ + 12: 'cookie1', + 34: 'cookie2', + }); + + delete process.env.BILIBILI_COOKIE_12; + delete process.env.BILIBILI_COOKIE_34; + }); + + it('email config', async () => { + process.env['EMAIL_CONFIG_xx.qq.com'] = 'token1'; + process.env['EMAIL_CONFIG_oo.qq.com'] = 'token2'; + + const { config } = await import('./config'); + expect(config.email.config).toMatchObject({ + 'xx.qq.com': 'token1', + 'oo.qq.com': 'token2', + }); + + delete process.env['EMAIL_CONFIG_xx.qq.com']; + delete process.env['EMAIL_CONFIG_oo.qq.com']; + }); + + it('discuz cookie', async () => { + process.env.DISCUZ_COOKIE_12 = 'cookie1'; + process.env.DISCUZ_COOKIE_34 = 'cookie2'; + + const { config } = await import('./config'); + expect(config.discuz.cookies).toMatchObject({ + 12: 'cookie1', + 34: 'cookie2', + }); + + delete process.env.DISCUZ_COOKIE_12; + delete process.env.DISCUZ_COOKIE_34; + }); + + it('medium cookie', async () => { + process.env.MEDIUM_COOKIE_12 = 'cookie1'; + process.env.MEDIUM_COOKIE_34 = 'cookie2'; + + const { config } = await import('./config'); + expect(config.medium.cookies).toMatchObject({ + 12: 'cookie1', + 34: 'cookie2', + }); + + delete process.env.MEDIUM_COOKIE_12; + delete process.env.MEDIUM_COOKIE_34; + }); + + it('discourse config', async () => { + process.env.DISCOURSE_CONFIG_12 = JSON.stringify({ a: 1 }); + process.env.DISCOURSE_CONFIG_34 = JSON.stringify({ b: 2 }); + + const { config } = await import('./config'); + expect(config.discourse.config).toMatchObject({ + 12: { a: 1 }, + 34: { b: 2 }, + }); + + delete process.env.DISCOURSE_CONFIG_12; + delete process.env.DISCOURSE_CONFIG_34; + }); + + it('no random ua', async () => { + process.env.NO_RANDOM_UA = '1'; + + const { config } = await import('./config'); + expect(config.ua).toBe('RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)'); + + delete process.env.NO_RANDOM_UA; + }); + + it('random ua', async () => { + const { config } = await import('./config'); + expect(config.ua).not.toBe('RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)'); + }); + + it('remote config', async () => { + process.env.REMOTE_CONFIG = 'http://rsshub.test/config'; + + const { config } = await import('./config'); + await new Promise((resolve) => setTimeout(resolve, 100)); + expect(config.ua).toBe('test'); + }); +}); diff --git a/lib/config.ts b/lib/config.ts new file mode 100644 index 00000000000000..ae084fab5cf184 --- /dev/null +++ b/lib/config.ts @@ -0,0 +1,860 @@ +import randUserAgent from '@/utils/rand-user-agent'; +import 'dotenv/config'; +import { ofetch } from 'ofetch'; + +let envs = process.env; + +export type Config = { + // app config + disallowRobot: boolean; + enableCluster?: string; + isPackage: boolean; + nodeName?: string; + puppeteerWSEndpoint?: string; + chromiumExecutablePath?: string; + // network + connect: { + port: number; + }; + listenInaddrAny: boolean; + requestRetry: number; + requestTimeout: number; + ua: string; + trueUA: string; + allowOrigin?: string; + // cache + cache: { + type: string; + requestTimeout: number; + routeExpire: number; + contentExpire: number; + }; + memory: { + max: number; + }; + redis: { + url: string; + }; + // proxy + proxyUri?: string; + proxy: { + protocol?: string; + host?: string; + port?: string; + auth?: string; + url_regex: string; + strategy: 'on_retry' | 'all'; + }; + pacUri?: string; + pacScript?: string; + // access control + accessKey?: string; + // logging + debugInfo: string; + loggerLevel: string; + noLogfiles?: boolean; + otel: { + seconds_bucket?: string; + milliseconds_bucket?: string; + }; + showLoggerTimestamp?: boolean; + sentry: { + dsn?: string; + routeTimeout: number; + }; + enableRemoteDebugging?: boolean; + // feed config + hotlink: { + template?: string; + includePaths?: string[]; + excludePaths?: string[]; + }; + feature: { + allow_user_hotlink_template: boolean; + filter_regex_engine: string; + allow_user_supply_unsafe_domain: boolean; + }; + suffix?: string; + titleLengthLimit: number; + openai: { + apiKey?: string; + model?: string; + temperature?: number; + maxTokens?: number; + endpoint: string; + inputOption: string; + promptTitle: string; + promptDescription: string; + }; + follow: { + ownerUserId?: string; + description?: string; + price?: number; + userLimit?: number; + }; + + // Route-specific Configurations + bilibili: { + cookies: Record; + dmImgList?: string; + dmImgInter?: string; + }; + bitbucket: { + username?: string; + password?: string; + }; + btbyr: { + host?: string; + cookies?: string; + }; + bupt: { + portal_cookie?: string; + }; + caixin: { + cookie?: string; + }; + civitai: { + cookie?: string; + }; + dianping: { + cookie?: string; + }; + dida365: { + username?: string; + password?: string; + }; + discord: { + authorization?: string; + }; + discourse: { + config: Record; + }; + discuz: { + cookies: Record; + }; + disqus: { + api_key?: string; + }; + douban: { + cookie?: string; + }; + ehentai: { + ipb_member_id?: string; + ipb_pass_hash?: string; + sk?: string; + igneous?: string; + star?: string; + img_proxy?: string; + }; + email: { + config: Record; + }; + fanbox: { + session?: string; + }; + fanfou: { + consumer_key?: string; + consumer_secret?: string; + username?: string; + password?: string; + }; + fantia: { + cookies?: string; + }; + game4399: { + cookie?: string; + }; + github: { + access_token?: string; + }; + gitee: { + access_token?: string; + }; + google: { + fontsApiKey?: string; + }; + guozaoke: { + cookies?: string; + }; + hefeng: { + key?: string; + }; + infzm: { + cookie?: string; + }; + initium: { + username?: string; + password?: string; + bearertoken?: string; + }; + instagram: { + username?: string; + password?: string; + proxy?: string; + cookie?: string; + }; + iwara: { + username?: string; + password?: string; + }; + javdb: { + session?: string; + }; + keylol: { + cookie?: string; + }; + lastfm: { + api_key?: string; + }; + lightnovel: { + cookie?: string; + }; + lorientlejour: { + token?: string; + username?: string; + password?: string; + }; + malaysiakini: { + email?: string; + password?: string; + refreshToken?: string; + }; + manhuagui: { + cookie?: string; + }; + mastodon: { + apiHost?: string; + accessToken?: string; + acctDomain?: string; + }; + medium: { + cookies: Record; + articleCookie?: string; + }; + mihoyo: { + cookie?: string; + }; + miniflux: { + instance?: string; + token?: string; + }; + mox: { + cookie: string; + }; + ncm: { + cookies?: string; + }; + newrank: { + cookie?: string; + }; + nga: { + uid?: string; + cid?: string; + }; + nhentai: { + username?: string; + password?: string; + }; + notion: { + key?: string; + }; + patreon: { + sessionId?: string; + }; + pianyuan: { + cookie?: string; + }; + pixabay: { + key?: string; + }; + pixiv: { + refreshToken?: string; + bypassCdn?: boolean; + bypassCdnHostname?: string; + bypassCdnDoh?: string; + imgProxy?: string; + }; + pkubbs: { + cookie?: string; + }; + qingting: { + id?: string; + }; + readwise: { + accessToken?: string; + }; + saraba1st: { + cookie?: string; + }; + sehuatang: { + cookie?: string; + }; + scboy: { + token?: string; + }; + scihub: { + host?: string; + }; + sis001: { + baseUrl?: string; + }; + skeb: { + bearerToken?: string; + }; + sorrycc: { + cookie?: string; + }; + spotify: { + clientId?: string; + clientSecret?: string; + refreshToken?: string; + }; + sspai: { + bearertoken?: string; + }; + telegram: { + token?: string; + session?: string; + apiId?: number; + apiHash?: string; + maxConcurrentDownloads?: number; + proxy?: { + host?: string; + port?: number; + secret?: string; + }; + }; + tophub: { + cookie?: string; + }; + tsdm39: { + cookie: string; + }; + twitter: { + username?: string[]; + password?: string[]; + authenticationSecret?: string[]; + phoneOrEmail?: string[]; + authToken?: string[]; + thirdPartyApi?: string; + }; + uestc: { + bbsCookie?: string; + bbsAuthStr?: string; + }; + weibo: { + app_key?: string; + app_secret?: string; + cookies?: string; + redirect_url?: string; + }; + wenku8: { + cookie?: string; + }; + wordpress: { + cdnUrl?: string; + }; + xiaoyuzhou: { + device_id?: string; + refresh_token?: string; + }; + xiaohongshu: { + cookie?: string; + }; + ximalaya: { + token?: string; + }; + xsijishe: { + cookie?: string; + userAgent?: string; + }; + xueqiu: { + cookies?: string; + }; + yamibo: { + salt?: string; + auth?: string; + }; + youtube: { + key?: string; + clientId?: string; + clientSecret?: string; + refreshToken?: string; + }; + zhihu: { + cookies?: string; + }; + zodgame: { + cookie?: string; + }; + zsxq: { + accessToken?: string; + }; +}; + +const value: Config | Record = {}; + +const TRUE_UA = 'RSSHub/1.0 (+http://github.com/DIYgod/RSSHub; like FeedFetcher-Google)'; + +const toBoolean = (value: string | undefined, defaultValue: boolean) => { + if (value === undefined) { + return defaultValue; + } else { + return value === '' || value === '0' || value === 'false' ? false : !!value; + } +}; + +const toInt = (value: string | undefined, defaultValue?: number) => (value === undefined ? defaultValue : Number.parseInt(value)); + +const calculateValue = () => { + const bilibili_cookies: Record = {}; + const email_config: Record = {}; + const discuz_cookies: Record = {}; + const medium_cookies: Record = {}; + const discourse_config: Record = {}; + + for (const name in envs) { + if (name.startsWith('BILIBILI_COOKIE_')) { + const uid = name.slice(16); + bilibili_cookies[uid] = envs[name]; + } else if (name.startsWith('EMAIL_CONFIG_')) { + const id = name.slice(13); + email_config[id] = envs[name]; + } else if (name.startsWith('DISCUZ_COOKIE_')) { + const cid = name.slice(14); + discuz_cookies[cid] = envs[name]; + } else if (name.startsWith('MEDIUM_COOKIE_')) { + const username = name.slice(14).toLowerCase(); + medium_cookies[username] = envs[name]; + } else if (name.startsWith('DISCOURSE_CONFIG_')) { + const id = name.slice('DISCOURSE_CONFIG_'.length); + discourse_config[id] = JSON.parse(envs[name] || '{}'); + } + } + + const _value = { + // app config + disallowRobot: toBoolean(envs.DISALLOW_ROBOT, false), + enableCluster: envs.ENABLE_CLUSTER, + isPackage: !!envs.IS_PACKAGE, + nodeName: envs.NODE_NAME, + puppeteerWSEndpoint: envs.PUPPETEER_WS_ENDPOINT, + chromiumExecutablePath: envs.CHROMIUM_EXECUTABLE_PATH, + // network + connect: { + port: toInt(envs.PORT, 1200), // 监听端口 + }, + listenInaddrAny: toBoolean(envs.LISTEN_INADDR_ANY, true), // 是否允许公网连接，取值 0 1 + requestRetry: toInt(envs.REQUEST_RETRY, 2), // 请求失败重试次数 + requestTimeout: toInt(envs.REQUEST_TIMEOUT, 30000), // Milliseconds to wait for the server to end the response before aborting the request + ua: envs.UA ?? (toBoolean(envs.NO_RANDOM_UA, false) ? TRUE_UA : randUserAgent({ browser: 'chrome', os: 'mac os', device: 'desktop' })), + trueUA: TRUE_UA, + allowOrigin: envs.ALLOW_ORIGIN, + // cache + cache: { + type: envs.CACHE_TYPE || (envs.CACHE_TYPE === '' ? '' : 'memory'), // 缓存类型，支持 'memory' 和 'redis'，设为空可以禁止缓存 + requestTimeout: toInt(envs.CACHE_REQUEST_TIMEOUT, 60), + routeExpire: toInt(envs.CACHE_EXPIRE, 5 * 60), // 路由缓存时间，单位为秒 + contentExpire: toInt(envs.CACHE_CONTENT_EXPIRE, 1 * 60 * 60), // 不变内容缓存时间，单位为秒 + }, + memory: { + max: toInt(envs.MEMORY_MAX, Math.pow(2, 8)), // The maximum number of items that remain in the cache. This must be a positive finite intger. + // https://github.com/isaacs/node-lru-cache#options + }, + redis: { + url: envs.REDIS_URL || 'redis://localhost:6379/', + }, + // proxy + proxyUri: envs.PROXY_URI, + proxy: { + protocol: envs.PROXY_PROTOCOL, + host: envs.PROXY_HOST, + port: envs.PROXY_PORT, + auth: envs.PROXY_AUTH, + url_regex: envs.PROXY_URL_REGEX || '.*', + strategy: envs.PROXY_STRATEGY || 'all', // all / on_retry + }, + pacUri: envs.PAC_URI, + pacScript: envs.PAC_SCRIPT, + // access control + accessKey: envs.ACCESS_KEY, + // logging + // 是否显示 Debug 信息，取值 'true' 'false' 'some_string' ，取值为 'true' 时永久显示，取值为 'false' 时永远隐藏，取值为 'some_string' 时请求带上 '?debug=some_string' 显示 + debugInfo: envs.DEBUG_INFO || 'true', + loggerLevel: envs.LOGGER_LEVEL || 'info', + noLogfiles: toBoolean(envs.NO_LOGFILES, false), + otel: { + seconds_bucket: envs.OTEL_SECONDS_BUCKET || '0.01,0.1,1,2,5,15,30,60', + milliseconds_bucket: envs.OTEL_MILLISECONDS_BUCKET || '10,20,50,100,250,500,1000,5000,15000', + }, + showLoggerTimestamp: toBoolean(envs.SHOW_LOGGER_TIMESTAMP, false), + sentry: { + dsn: envs.SENTRY, + routeTimeout: toInt(envs.SENTRY_ROUTE_TIMEOUT, 30000), + }, + enableRemoteDebugging: toBoolean(envs.ENABLE_REMOTE_DEBUGGING, false), + // feed config + hotlink: { + template: envs.HOTLINK_TEMPLATE, + includePaths: envs.HOTLINK_INCLUDE_PATHS ? envs.HOTLINK_INCLUDE_PATHS.split(',') : undefined, + excludePaths: envs.HOTLINK_EXCLUDE_PATHS ? envs.HOTLINK_EXCLUDE_PATHS.split(',') : undefined, + }, + feature: { + allow_user_hotlink_template: toBoolean(envs.ALLOW_USER_HOTLINK_TEMPLATE, false), + filter_regex_engine: envs.FILTER_REGEX_ENGINE || 're2', + allow_user_supply_unsafe_domain: toBoolean(envs.ALLOW_USER_SUPPLY_UNSAFE_DOMAIN, false), + }, + suffix: envs.SUFFIX, + titleLengthLimit: toInt(envs.TITLE_LENGTH_LIMIT, 150), + openai: { + apiKey: envs.OPENAI_API_KEY, + model: envs.OPENAI_MODEL || 'gpt-3.5-turbo-16k', + temperature: toInt(envs.OPENAI_TEMPERATURE, 0.2), + maxTokens: toInt(envs.OPENAI_MAX_TOKENS, 0) || undefined, + endpoint: envs.OPENAI_API_ENDPOINT || 'https://api.openai.com/v1', + inputOption: envs.OPENAI_INPUT_OPTION || 'description', + promptDescription: envs.OPENAI_PROMPT || 'Please summarize the following article and reply with markdown format.', + promptTitle: envs.OPENAI_PROMPT_TITLE || 'Please translate the following title into Simplified Chinese and reply only translated text.', + }, + follow: { + ownerUserId: envs.FOLLOW_OWNER_USER_ID, + description: envs.FOLLOW_DESCRIPTION, + price: toInt(envs.FOLLOW_PRICE), + userLimit: toInt(envs.FOLLOW_USER_LIMIT), + }, + + // Route-specific Configurations + bilibili: { + cookies: bilibili_cookies, + dmImgList: envs.BILIBILI_DM_IMG_LIST, + dmImgInter: envs.BILIBILI_DM_IMG_INTER, + }, + bitbucket: { + username: envs.BITBUCKET_USERNAME, + password: envs.BITBUCKET_PASSWORD, + }, + btbyr: { + host: envs.BTBYR_HOST, + cookies: envs.BTBYR_COOKIE, + }, + bupt: { + portal_cookie: envs.BUPT_PORTAL_COOKIE, + }, + caixin: { + cookie: envs.CAIXIN_COOKIE, + }, + civitai: { + cookie: envs.CIVITAI_COOKIE, + }, + dianping: { + cookie: envs.DIANPING_COOKIE, + }, + dida365: { + username: envs.DIDA365_USERNAME, + password: envs.DIDA365_PASSWORD, + }, + discord: { + authorization: envs.DISCORD_AUTHORIZATION, + }, + discourse: { + config: discourse_config, + }, + discuz: { + cookies: discuz_cookies, + }, + disqus: { + api_key: envs.DISQUS_API_KEY, + }, + douban: { + cookie: envs.DOUBAN_COOKIE, + }, + ehentai: { + ipb_member_id: envs.EH_IPB_MEMBER_ID, + ipb_pass_hash: envs.EH_IPB_PASS_HASH, + sk: envs.EH_SK, + igneous: envs.EH_IGNEOUS, + star: envs.EH_STAR, + img_proxy: envs.EH_IMG_PROXY, + }, + email: { + config: email_config, + }, + fanbox: { + session: envs.FANBOX_SESSION_ID, + }, + fanfou: { + consumer_key: envs.FANFOU_CONSUMER_KEY, + consumer_secret: envs.FANFOU_CONSUMER_SECRET, + username: envs.FANFOU_USERNAME, + password: envs.FANFOU_PASSWORD, + }, + fantia: { + cookies: envs.FANTIA_COOKIE, + }, + game4399: { + cookie: envs.GAME_4399, + }, + github: { + access_token: envs.GITHUB_ACCESS_TOKEN, + }, + gitee: { + access_token: envs.GITEE_ACCESS_TOKEN, + }, + google: { + fontsApiKey: envs.GOOGLE_FONTS_API_KEY, + }, + guozaoke: { + cookies: envs.GUOZAOKE_COOKIES, + }, + hefeng: { + // weather + key: envs.HEFENG_KEY, + }, + infzm: { + cookie: envs.INFZM_COOKIE, + }, + initium: { + username: envs.INITIUM_USERNAME, + password: envs.INITIUM_PASSWORD, + bearertoken: envs.INITIUM_BEARER_TOKEN, + }, + instagram: { + username: envs.IG_USERNAME, + password: envs.IG_PASSWORD, + proxy: envs.IG_PROXY, + cookie: envs.IG_COOKIE, + }, + iwara: { + username: envs.IWARA_USERNAME, + password: envs.IWARA_PASSWORD, + }, + javdb: { + session: envs.JAVDB_SESSION, + }, + keylol: { + cookie: envs.KEYLOL_COOKIE, + }, + lastfm: { + api_key: envs.LASTFM_API_KEY, + }, + lightnovel: { + cookie: envs.SECURITY_KEY, + }, + lorientlejour: { + token: envs.LORIENTLEJOUR_TOKEN, + username: envs.LORIENTLEJOUR_USERNAME, + password: envs.LORIENTLEJOUR_PASSWORD, + }, + malaysiakini: { + email: envs.MALAYSIAKINI_EMAIL, + password: envs.MALAYSIAKINI_PASSWORD, + refreshToken: envs.MALAYSIAKINI_REFRESHTOKEN, + }, + manhuagui: { + cookie: envs.MHGUI_COOKIE, + }, + mastodon: { + apiHost: envs.MASTODON_API_HOST, + accessToken: envs.MASTODON_API_ACCESS_TOKEN, + acctDomain: envs.MASTODON_API_ACCT_DOMAIN, + }, + medium: { + cookies: medium_cookies, + articleCookie: envs.MEDIUM_ARTICLE_COOKIE || '', + }, + mihoyo: { + cookie: envs.MIHOYO_COOKIE, + }, + miniflux: { + instance: envs.MINIFLUX_INSTANCE || 'https://reader.miniflux.app', + token: envs.MINIFLUX_TOKEN || '', + }, + mox: { + cookie: envs.MOX_COOKIE, + }, + ncm: { + cookies: envs.NCM_COOKIES || '', + }, + newrank: { + cookie: envs.NEWRANK_COOKIE, + }, + nga: { + uid: envs.NGA_PASSPORT_UID, + cid: envs.NGA_PASSPORT_CID, + }, + nhentai: { + username: envs.NHENTAI_USERNAME, + password: envs.NHENTAI_PASSWORD, + }, + notion: { + key: envs.NOTION_TOKEN, + }, + patreon: { + sessionId: envs.PATREON_SESSION_ID, + }, + pianyuan: { + cookie: envs.PIANYUAN_COOKIE, + }, + pixabay: { + key: envs.PIXABAY_KEY, + }, + pixiv: { + refreshToken: envs.PIXIV_REFRESHTOKEN, + bypassCdn: toBoolean(envs.PIXIV_BYPASS_CDN, false), + bypassCdnHostname: envs.PIXIV_BYPASS_HOSTNAME || 'public-api.secure.pixiv.net', + bypassCdnDoh: envs.PIXIV_BYPASS_DOH || 'https://1.1.1.1/dns-query', + imgProxy: envs.PIXIV_IMG_PROXY || 'https://i.pixiv.re', + }, + pkubbs: { + cookie: envs.PKUBBS_COOKIE, + }, + qingting: { + id: envs.QINGTING_ID, + }, + readwise: { + accessToken: envs.READWISE_ACCESS_TOKEN, + }, + saraba1st: { + cookie: envs.SARABA1ST_COOKIE, + }, + sehuatang: { + cookie: envs.SEHUATANG_COOKIE, + }, + scboy: { + token: envs.SCBOY_BBS_TOKEN, + }, + scihub: { + host: envs.SCIHUB_HOST || 'https://sci-hub.se/', + }, + sis001: { + baseUrl: envs.SIS001_BASE_URL || 'https://sis001.com', + }, + skeb: { + bearerToken: envs.SKEB_BEARER_TOKEN, + }, + sorrycc: { + cookie: envs.SORRYCC_COOKIES, + }, + spotify: { + clientId: envs.SPOTIFY_CLIENT_ID, + clientSecret: envs.SPOTIFY_CLIENT_SECRET, + refreshToken: envs.SPOTIFY_REFRESHTOKEN, + }, + sspai: { + bearertoken: envs.SSPAI_BEARERTOKEN, + }, + telegram: { + token: envs.TELEGRAM_TOKEN, + session: envs.TELEGRAM_SESSION, + apiId: envs.TELEGRAM_API_ID, + apiHash: envs.TELEGRAM_API_HASH, + maxConcurrentDownloads: envs.TELEGRAM_MAX_CONCURRENT_DOWNLOADS, + proxy: { + host: envs.TELEGRAM_PROXY_HOST, + port: envs.TELEGRAM_PROXY_PORT, + secret: envs.TELEGRAM_PROXY_SECRET, + }, + }, + tophub: { + cookie: envs.TOPHUB_COOKIE, + }, + tsdm39: { + cookie: envs.TSDM39_COOKIES, + }, + twitter: { + username: envs.TWITTER_USERNAME?.split(','), + password: envs.TWITTER_PASSWORD?.split(','), + authenticationSecret: envs.TWITTER_AUTHENTICATION_SECRET?.split(','), + phoneOrEmail: envs.TWITTER_PHONE_OR_EMAIL?.split(','), + authToken: envs.TWITTER_AUTH_TOKEN?.split(','), + thirdPartyApi: envs.TWITTER_THIRD_PARTY_API, + }, + uestc: { + bbsCookie: envs.UESTC_BBS_COOKIE, + bbsAuthStr: envs.UESTC_BBS_AUTH_STR, + }, + weibo: { + app_key: envs.WEIBO_APP_KEY, + app_secret: envs.WEIBO_APP_SECRET, + cookies: envs.WEIBO_COOKIES, + redirect_url: envs.WEIBO_REDIRECT_URL, + }, + wenku8: { + cookie: envs.WENKU8_COOKIE, + }, + wordpress: { + cdnUrl: envs.WORDPRESS_CDN, + }, + xiaoyuzhou: { + device_id: envs.XIAOYUZHOU_ID, + refresh_token: envs.XIAOYUZHOU_TOKEN, + }, + xiaohongshu: { + cookie: envs.XIAOHONGSHU_COOKIE, + }, + ximalaya: { + token: envs.XIMALAYA_TOKEN, + }, + xsijishe: { + cookie: envs.XSIJISHE_COOKIE, + user_agent: envs.XSIJISHE_USER_AGENT, + }, + xueqiu: { + cookies: envs.XUEQIU_COOKIES, + }, + yamibo: { + salt: envs.YAMIBO_SALT, + auth: envs.YAMIBO_AUTH, + }, + youtube: { + key: envs.YOUTUBE_KEY, + clientId: envs.YOUTUBE_CLIENT_ID, + clientSecret: envs.YOUTUBE_CLIENT_SECRET, + refreshToken: envs.YOUTUBE_REFRESH_TOKEN, + }, + zhihu: { + cookies: envs.ZHIHU_COOKIES, + }, + zodgame: { + cookie: envs.ZODGAME_COOKIE, + }, + zsxq: { + accessToken: envs.ZSXQ_ACCESS_TOKEN, + }, + }; + + for (const name in _value) { + value[name] = _value[name]; + } +}; +calculateValue(); + +(async () => { + if (envs.REMOTE_CONFIG) { + const { default: logger } = await import('@/utils/logger'); + try { + const data = await ofetch(envs.REMOTE_CONFIG, { + headers: { + Authorization: `Basic ${envs.REMOTE_CONFIG_AUTH}`, + }, + }); + if (data) { + envs = Object.assign(envs, data); + calculateValue(); + logger.info('Remote config loaded.'); + } else { + logger.error('Remote config load failed.'); + } + } catch (error) { + logger.error('Remote config load failed.', error); + } + } +})(); + +// @ts-expect-error value is set +export const config: Config = value; + +export const setConfig = (env: Record) => { + envs = Object.assign(process.env, env); + calculateValue(); +}; diff --git a/lib/core_router.js b/lib/core_router.js deleted file mode 100644 index 823485a702f7e7..00000000000000 --- a/lib/core_router.js +++ /dev/null @@ -1,17 +0,0 @@ -const Router = require('@koa/router'); -const config = require('@/config').value; -const router = new Router(); - -// Load Core Route -router.get('/', require('./routes/index')); - -router.get('/robots.txt', (ctx) => { - if (config.disallowRobot) { - ctx.set('Content-Type', 'text/plain'); - ctx.body = 'User-agent: *\nDisallow: /'; - } else { - ctx.throw(404, 'Not Found'); - } -}); - -module.exports = router; diff --git a/lib/errors/RequestInProgress.js b/lib/errors/RequestInProgress.js deleted file mode 100644 index 222be7e91d41a3..00000000000000 --- a/lib/errors/RequestInProgress.js +++ /dev/null @@ -1,3 +0,0 @@ -class RequestInProgressError extends Error {} - -module.exports = RequestInProgressError; diff --git a/lib/errors/index.js b/lib/errors/index.js deleted file mode 100644 index 63fe1f2820f9d8..00000000000000 --- a/lib/errors/index.js +++ /dev/null @@ -1,3 +0,0 @@ -module.exports = { - RequestInProgressError: require('./RequestInProgress'), -}; diff --git a/lib/errors/index.test.ts b/lib/errors/index.test.ts new file mode 100644 index 00000000000000..e60fa114ac8328 --- /dev/null +++ b/lib/errors/index.test.ts @@ -0,0 +1,88 @@ +import { describe, expect, it, afterAll } from 'vitest'; +import supertest from 'supertest'; +import server from '@/index'; +import { load } from 'cheerio'; +import { config } from '@/config'; + +const request = supertest(server); + +afterAll(() => { + server.close(); +}); + +describe('error', () => { + it(`error`, async () => { + const response = await request.get('/test/error'); + expect(response.status).toBe(503); + expect(response.text).toMatch(/Error: Error test/); + }); +}); + +describe('httperror', () => { + it(`httperror`, async () => { + const response = await request.get('/test/httperror'); + expect(response.status).toBe(503); + expect(response.text).toContain('FetchError: [GET] "https://httpbingo.org/status/404": 404 Not Found'); + }, 20000); +}); + +describe('RequestInProgressError', () => { + it(`RequestInProgressError with retry`, async () => { + const responses = await Promise.all([request.get('/test/slow'), request.get('/test/slow')]); + expect(new Set(responses.map((r) => r.status))).toEqual(new Set([200, 200])); + }); + it(`RequestInProgressError`, async () => { + const responses = await Promise.all([request.get('/test/slow4'), request.get('/test/slow4')]); + expect(new Set(responses.map((r) => r.status))).toEqual(new Set([200, 503])); + expect(new Set(responses.map((r) => r.headers['cache-control']))).toEqual(new Set([`public, max-age=${config.cache.routeExpire}`, `public, max-age=${config.requestTimeout / 1000}`])); + expect(responses.filter((r) => r.text.includes('RequestInProgressError: This path is currently fetching, please come back later!'))).toHaveLength(1); + }); +}); + +describe('config-not-found-error', () => { + it(`config-not-found-error`, async () => { + const response = await request.get('/test/config-not-found-error'); + expect(response.status).toBe(503); + expect(response.text).toMatch('ConfigNotFoundError: Test config not found error'); + }, 20000); +}); + +describe('invalid-parameter-error', () => { + it(`invalid-parameter-error`, async () => { + const response = await request.get('/test/invalid-parameter-error'); + expect(response.status).toBe(503); + expect(response.text).toMatch('InvalidParameterError: Test invalid parameter error'); + }, 20000); +}); + +describe('route throws an error', () => { + it('route path error should have path mounted', async () => { + await request.get('/test/error'); + await request.get('/thisDoesNotExist'); + const response = await request.get('/'); + + const $ = load(response.text); + $('.debug-item').each((index, item) => { + const key = $(item).find('.debug-key').text().trim(); + const value = $(item).find('.debug-value').html()?.trim(); + switch (key) { + case 'Request Amount:': + expect(value).toBe('11'); + break; + case 'Hot Routes:': + expect(value).toBe('8 /test/:id/:params?
'); + break; + case 'Hot Paths:': + expect(value).toBe('2 /test/error
2 /test/slow
2 /test/slow4
1 /test/httperror
1 /test/config-not-found-error
1 /test/invalid-parameter-error
1 /thisDoesNotExist
1 /
'); + break; + case 'Hot Error Routes:': + expect(value).toBe('5 /test/:id/:params?
'); + break; + case 'Hot Error Paths:': + expect(value).toBe('2 /test/error
1 /test/httperror
1 /test/slow4
1 /test/config-not-found-error
1 /test/invalid-parameter-error
1 /thisDoesNotExist
'); + break; + default: + } + }); + }); +}); diff --git a/lib/errors/index.tsx b/lib/errors/index.tsx new file mode 100644 index 00000000000000..0ac4027939b322 --- /dev/null +++ b/lib/errors/index.tsx @@ -0,0 +1,81 @@ +import { type NotFoundHandler, type ErrorHandler } from 'hono'; +import { getDebugInfo, setDebugInfo } from '@/utils/debug-info'; +import { config } from '@/config'; +import * as Sentry from '@sentry/node'; +import logger from '@/utils/logger'; +import Error from '@/views/error'; + +import NotFoundError from './types/not-found'; + +import { requestMetric } from '@/utils/otel'; + +export const errorHandler: ErrorHandler = (error, ctx) => { + const requestPath = ctx.req.path; + const matchedRoute = ctx.req.routePath; + const hasMatchedRoute = matchedRoute !== '/*'; + + const debug = getDebugInfo(); + try { + if (ctx.res.headers.get('RSSHub-Cache-Status')) { + debug.hitCache++; + } + } catch { + // ignore + } + debug.error++; + + if (!debug.errorPaths[requestPath]) { + debug.errorPaths[requestPath] = 0; + } + debug.errorPaths[requestPath]++; + + if (!debug.errorRoutes[matchedRoute] && hasMatchedRoute) { + debug.errorRoutes[matchedRoute] = 0; + } + hasMatchedRoute && debug.errorRoutes[matchedRoute]++; + setDebugInfo(debug); + + if (config.sentry.dsn) { + Sentry.withScope((scope) => { + scope.setTag('name', requestPath.split('/')[1]); + Sentry.captureException(error); + }); + } + + let errorMessage = process.env.NODE_ENV === 'production' ? error.message : error.stack || error.message; + switch (error.constructor.name) { + case 'HTTPError': + case 'RequestError': + case 'FetchError': + ctx.status(503); + break; + case 'RequestInProgressError': + ctx.header('Cache-Control', `public, max-age=${config.requestTimeout / 1000}`); + ctx.status(503); + break; + case 'RejectError': + ctx.status(403); + break; + case 'NotFoundError': + ctx.status(404); + errorMessage += 'The route does not exist or has been deleted.'; + break; + default: + ctx.status(503); + break; + } + const message = `${error.name}: ${errorMessage}`; + + logger.error(`Error in ${requestPath}: ${message}`); + requestMetric.error({ path: matchedRoute, method: ctx.req.method, status: ctx.res.status }); + + return config.isPackage || ctx.req.query('format') === 'json' + ? ctx.json({ + error: { + message: error.message ?? error, + }, + }) + : ctx.html(); +}; + +export const notFoundHandler: NotFoundHandler = (ctx) => errorHandler(new NotFoundError(), ctx); diff --git a/lib/errors/types/config-not-found.ts b/lib/errors/types/config-not-found.ts new file mode 100644 index 00000000000000..c96cea03c2b80e --- /dev/null +++ b/lib/errors/types/config-not-found.ts @@ -0,0 +1,5 @@ +class ConfigNotFoundError extends Error { + name = 'ConfigNotFoundError'; +} + +export default ConfigNotFoundError; diff --git a/lib/errors/types/invalid-parameter.ts b/lib/errors/types/invalid-parameter.ts new file mode 100644 index 00000000000000..8599ec4d2af4f3 --- /dev/null +++ b/lib/errors/types/invalid-parameter.ts @@ -0,0 +1,5 @@ +class InvalidParameterError extends Error { + name = 'InvalidParameterError'; +} + +export default InvalidParameterError; diff --git a/lib/errors/types/not-found.ts b/lib/errors/types/not-found.ts new file mode 100644 index 00000000000000..9cba16b31e797a --- /dev/null +++ b/lib/errors/types/not-found.ts @@ -0,0 +1,5 @@ +class NotFoundError extends Error { + name = 'NotFoundError'; +} + +export default NotFoundError; diff --git a/lib/errors/types/reject.ts b/lib/errors/types/reject.ts new file mode 100644 index 00000000000000..b6b91fe4967c4a --- /dev/null +++ b/lib/errors/types/reject.ts @@ -0,0 +1,5 @@ +class RejectError extends Error { + name = 'RejectError'; +} + +export default RejectError; diff --git a/lib/errors/types/request-in-progress.ts b/lib/errors/types/request-in-progress.ts new file mode 100644 index 00000000000000..73ae4b5705d37c --- /dev/null +++ b/lib/errors/types/request-in-progress.ts @@ -0,0 +1,5 @@ +class RequestInProgressError extends Error { + name = 'RequestInProgressError'; +} + +export default RequestInProgressError; diff --git a/lib/index.js b/lib/index.js deleted file mode 100644 index cf7e0f63fc0097..00000000000000 --- a/lib/index.js +++ /dev/null @@ -1,35 +0,0 @@ -const app = require('./app'); -const config = require('./config').value; -const fs = require('fs'); -const logger = require('./utils/logger'); - -const cluster = require('cluster'); -const numCPUs = require('os').cpus().length; - -if (config.enableCluster && cluster.isMaster && process.env.NODE_ENV !== 'test' && process.env.NODE_ENV !== 'dev') { - for (let i = 0; i < numCPUs; i++) { - cluster.fork(); - } -} else { - let server; - if (config.connect.socket) { - if (fs.existsSync(config.connect.socket)) { - fs.unlinkSync(config.connect.socket); - } - server = app.listen(config.connect.socket, parseInt(config.listenInaddrAny) ? null : '127.0.0.1'); - logger.info('Listening Unix Socket ' + config.connect.socket); - process.on('SIGINT', () => { - fs.unlinkSync(config.connect.socket); - process.exit(); - }); - } - if (config.connect.port) { - server = app.listen(config.connect.port, parseInt(config.listenInaddrAny) ? null : '127.0.0.1'); - logger.info('Listening Port ' + config.connect.port); - } - - logger.info('🎉 RSSHub start! Cheers!'); - logger.info('💖 Can you help keep this open source project alive? Please sponsor 👉 https://docs.rsshub.app/support'); - - module.exports = server; -} diff --git a/lib/index.ts b/lib/index.ts new file mode 100644 index 00000000000000..61fbec2f9311f2 --- /dev/null +++ b/lib/index.ts @@ -0,0 +1,28 @@ +import { serve } from '@hono/node-server'; +import logger from '@/utils/logger'; +import { getLocalhostAddress } from '@/utils/common-utils'; +import { config } from '@/config'; +import app from '@/app'; + +const port = config.connect.port; +const hostIPList = getLocalhostAddress(); + +logger.info(`🎉 RSSHub is running on port ${port}! Cheers!`); +logger.info('💖 Can you help keep this open source project alive? Please sponsor 👉 https://docs.rsshub.app/sponsor'); +logger.info(`🔗 Local: 👉 http://localhost:${port}`); +if (config.listenInaddrAny) { + for (const ip of hostIPList) { + logger.info(`🔗 Network: 👉 http://${ip}:${port}`); + } +} + +const server = serve({ + fetch: app.fetch, + hostname: config.listenInaddrAny ? '::' : '127.0.0.1', + port, + serverOptions: { + maxHeaderSize: 1024 * 32, + }, +}); + +export default server; diff --git a/lib/maintainer.js b/lib/maintainer.js deleted file mode 100644 index 9efb9f448b62d8..00000000000000 --- a/lib/maintainer.js +++ /dev/null @@ -1,52 +0,0 @@ -const dirname = __dirname + '/v2'; -const fs = require('fs'); -const { join } = require('path'); - -// Presence Check -for (const dir of fs.readdirSync(dirname)) { - const dirPath = join(dirname, dir); - if (fs.existsSync(join(dirPath, 'router.js')) && !fs.existsSync(join(dirPath, 'maintainer.js'))) { - throw Error(`No maintainer.js in "${dirPath}".`); - } -} - -// 遍历整个 routes 文件夹，收集模块 maintainer.js -const maintainerPath = require('require-all')({ - dirname, - filter: /maintainer\.js$/, -}); - -const maintainers = {}; - -// 将收集到的自定义模块进行合并 -for (const dir in maintainerPath) { - const routes = maintainerPath[dir]['maintainer.js']; // Do not merge other file - - // typo check e.g., ✘ module.export, ✔ module.exports - if (!Object.keys(routes).length) { - throw Error(`No maintainer in "${dir}".`); - } - for (const author of Object.values(routes)) { - if (!Array.isArray(author)) { - throw Error(`Maintainers' name should be an array in "${dir}".`); - } - // check for [], [''] or ['Someone', ''] - if (author.length < 1 || author.includes('')) { - throw Error(`Empty maintainer in "${dir}".`); - } - } - - for (const key in routes) { - maintainers['/' + dir + key] = routes[key]; - } -} - -// 兼容旧版路由 -const router = require('./router'); -router.stack.forEach((e) => { - if (!maintainers[e.path]) { - maintainers[e.path] = []; - } -}); - -module.exports = maintainers; diff --git a/lib/middleware/access-control.js b/lib/middleware/access-control.js deleted file mode 100644 index 85d05936972504..00000000000000 --- a/lib/middleware/access-control.js +++ /dev/null @@ -1,55 +0,0 @@ -const config = require('@/config').value; -const md5 = require('@/utils/md5'); -const isLocalhost = require('is-localhost-ip'); - -const reject = (ctx) => { - ctx.response.status = 403; - - throw Error('Authentication failed. Access denied.'); -}; - -module.exports = async (ctx, next) => { - const ip = ctx.ips[0] || ctx.ip; - const requestPath = ctx.request.path; - const requestUA = ctx.request.header['user-agent']; - const accessKey = ctx.query.key; - const accessCode = ctx.query.code; - - const isControlled = config.accessKey || config.whitelist || config.blacklist; - - const allowLocalhost = config.allowLocalhost && (await isLocalhost(ip)); - - const grant = async () => { - if (ctx.response.status !== 403) { - await next(); - } - }; - - if (requestPath === '/' || requestPath === '/robots.txt') { - await next(); - } else { - if (!isControlled || allowLocalhost) { - return grant(); - } - - if (config.accessKey) { - if (config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey)) { - return grant(); - } - } - - if (config.whitelist) { - if (config.whitelist.find((white) => ip.includes(white) || requestPath.includes(white) || requestUA.includes(white))) { - return grant(); - } - } - - if (config.blacklist) { - if (!config.blacklist.find((black) => ip.includes(black) || requestPath.includes(black) || requestUA.includes(black))) { - return grant(); - } - } - - reject(ctx); - } -}; diff --git a/lib/middleware/access-control.test.ts b/lib/middleware/access-control.test.ts new file mode 100644 index 00000000000000..83dd5be0f5dea7 --- /dev/null +++ b/lib/middleware/access-control.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, it, vi, afterEach } from 'vitest'; +import md5 from '@/utils/md5'; + +process.env.NODE_NAME = 'mock'; + +async function checkBlock(response) { + expect(response.status).toBe(403); + expect(await response.text()).toMatch(/Access denied\./); +} + +afterEach(() => { + delete process.env.ACCESS_KEY; + vi.resetModules(); +}); + +describe('access-control', () => { + it(`access key`, async () => { + const key = '1L0veRSSHub'; + const code = md5('/test/2' + key); + process.env.ACCESS_KEY = key; + const app = (await import('@/app')).default; + + const response01 = await app.request('/'); + expect(response01.status).toBe(200); + + const response02 = await app.request('/robots.txt'); + expect(response02.status).toBe(404); + + // no key/code + const response21 = await app.request('/test/2'); + await checkBlock(response21); + + // wrong key/code + const response321 = await app.request(`/test/2?key=wrong+${key}`); + await checkBlock(response321); + + const response322 = await app.request(`/test/2?code=wrong+${code}`); + await checkBlock(response322); + + // right key/code + const response331 = await app.request(`/test/2?key=${key}`); + expect(response331.status).toBe(200); + + const response332 = await app.request(`/test/2?code=${code}`); + expect(response332.status).toBe(200); + }); +}); diff --git a/lib/middleware/access-control.ts b/lib/middleware/access-control.ts new file mode 100644 index 00000000000000..41123b1f84527a --- /dev/null +++ b/lib/middleware/access-control.ts @@ -0,0 +1,25 @@ +import type { MiddlewareHandler } from 'hono'; +import { config } from '@/config'; +import md5 from '@/utils/md5'; +import RejectError from '@/errors/types/reject'; + +const reject = (requestPath) => { + throw new RejectError(`Authentication failed. Access denied.\n${requestPath}`); +}; + +const middleware: MiddlewareHandler = async (ctx, next) => { + const requestPath = new URL(ctx.req.url).pathname; + const accessKey = ctx.req.query('key'); + const accessCode = ctx.req.query('code'); + + if (requestPath === '/' || requestPath === '/robots.txt' || requestPath === '/favicon.ico' || requestPath === '/logo.png') { + await next(); + } else { + if (config.accessKey && !(config.accessKey === accessKey || accessCode === md5(requestPath + config.accessKey))) { + return reject(requestPath); + } + await next(); + } +}; + +export default middleware; diff --git a/lib/middleware/anti-hotlink.js b/lib/middleware/anti-hotlink.js deleted file mode 100644 index 04948701009cda..00000000000000 --- a/lib/middleware/anti-hotlink.js +++ /dev/null @@ -1,148 +0,0 @@ -const config = require('@/config').value; -const cheerio = require('cheerio'); -const logger = require('@/utils/logger'); -const path = require('path'); -const { art } = require('@/utils/render'); - -const templateRegex = /\$\{([^{}]+)}/g; -const allowedUrlProperties = ['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username']; -const IframeWrapperTemplate = path.join(__dirname, 'templates/iframe.art'); - -// match path or sub-path -const matchPath = (path, paths) => { - for (const p of paths) { - if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) { - return true; - } - } - return false; -}; - -// return ture if the path needs to be processed -const filterPath = (path) => { - const include = config.hotlink.includePaths; - const exclude = config.hotlink.excludePaths; - return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude)); -}; - -const interpolate = (str, obj) => - str.replace(templateRegex, (_, prop) => { - let needEncode = false; - if (prop.endsWith('_ue')) { - // url encode - prop = prop.slice(0, -3); - needEncode = true; - } - return needEncode ? encodeURIComponent(obj[prop]) : obj[prop]; - }); -const parseUrl = (str) => { - let url; - try { - url = new URL(str); - } catch (e) { - logger.error(`Failed to parse ${str}`); - } - - return url; -}; -const replaceUrls = ($, selector, template, attribute = 'src') => { - $(selector).each(function () { - const old_src = $(this).attr(attribute); - if (old_src) { - const url = parseUrl(old_src); - if (url) { - // Cheerio will do the right thing to prohibit XSS. - $(this).attr(attribute, interpolate(template, url)); - } - } - }); -}; - -const wrapWithIframe = ($, selector) => { - $(selector).each((_, elem) => { - elem = $(elem); - elem.replaceWith(art(IframeWrapperTemplate, { content: elem.toString() })); - }); -}; - -const process = (html, image_hotlink_template, multimedia_hotlink_template, wrap_multimedia_in_iframe) => { - const $ = cheerio.load(html, undefined, false); - if (image_hotlink_template) { - replaceUrls($, 'img, picture > source', image_hotlink_template); - replaceUrls($, 'video[poster]', image_hotlink_template, 'poster'); - } - if (multimedia_hotlink_template) { - replaceUrls($, 'video, video > source, audio, audio > source', multimedia_hotlink_template); - if (!image_hotlink_template) { - replaceUrls($, 'video[poster]', multimedia_hotlink_template, 'poster'); - } - } - if (wrap_multimedia_in_iframe) { - wrapWithIframe($, 'video, audio'); - } - return $.html(); -}; - -const validateTemplate = (template) => { - if (!template) { - return; - } - [...template.matchAll(templateRegex)].forEach((match) => { - const prop = match[1].endsWith('_ue') ? match[1].slice(0, -3) : match[1]; - if (!allowedUrlProperties.includes(prop)) { - throw new Error(`Invalid URL property: ${prop}`); - } - }); -}; - -module.exports = async (ctx, next) => { - await next(); - - let image_hotlink_template; - let multimedia_hotlink_template; - const shouldWrapInIframe = ctx.query.wrap_multimedia_in_iframe === '1'; - - // Read params if enabled - if (config.feature.allow_user_hotlink_template) { - // By default, the config turns these features off. Set corresponding config to - // true to turn this feature on. - // A risk is that the media URLs will be replaced by user-supplied templates, - // so a user could literally take the control of "where are the media from", - // but only in their personal-use feed URL. - multimedia_hotlink_template = ctx.query.multimedia_hotlink_template; - image_hotlink_template = ctx.query.image_hotlink_template; - } - - // Force config hotlink template on conflict - if (config.hotlink.template) { - if (!filterPath(ctx.request.path)) { - image_hotlink_template = undefined; - } else { - image_hotlink_template = config.hotlink.template; - } - } - - if (!image_hotlink_template && !multimedia_hotlink_template && !shouldWrapInIframe) { - return; - } - - validateTemplate(image_hotlink_template); - validateTemplate(multimedia_hotlink_template); - - // Assume that only description include image link - // and here we will only check them in description. - // Use Cheerio to load the description as html and filter all - // image link - if (ctx.state.data) { - if (ctx.state.data.description) { - ctx.state.data.description = process(ctx.state.data.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe); - } - - ctx.state.data.item && - ctx.state.data.item.forEach((item) => { - if (item.description) { - item.description = process(item.description, image_hotlink_template, multimedia_hotlink_template, shouldWrapInIframe); - } - }); - } -}; diff --git a/lib/middleware/anti-hotlink.test.ts b/lib/middleware/anti-hotlink.test.ts new file mode 100644 index 00000000000000..70c30f1343b2ea --- /dev/null +++ b/lib/middleware/anti-hotlink.test.ts @@ -0,0 +1,440 @@ +import { describe, expect, it, vi, afterEach, afterAll } from 'vitest'; +import Parser from 'rss-parser'; + +const parser = new Parser(); + +afterAll(() => { + delete process.env.HOTLINK_TEMPLATE; + delete process.env.HOTLINK_INCLUDE_PATHS; + delete process.env.HOTLINK_EXCLUDE_PATHS; + delete process.env.ALLOW_USER_HOTLINK_TEMPLATE; +}); + +afterEach(() => { + delete process.env.HOTLINK_TEMPLATE; + delete process.env.HOTLINK_INCLUDE_PATHS; + delete process.env.HOTLINK_EXCLUDE_PATHS; + delete process.env.ALLOW_USER_HOTLINK_TEMPLATE; + vi.resetModules(); +}); + +const expects = { + complicated: { + origin: { + items: [ + ` + + + + + + + + + +`, + ` +`, + ], + desc: ' - Powered by RSSHub', + }, + processed: { + items: [ + ` + + + + + + + + + +`, + ` +`, + ], + desc: ' - Powered by RSSHub', + }, + urlencoded: { + items: [ + ` + + + + + + + + + +`, + ` +`, + ], + desc: ' - Powered by RSSHub', + }, + }, + multimedia: { + origin: { + items: [ + ` + + + +`, + ], + desc: ' - Powered by RSSHub', + }, + relayed: { + items: [ + ` + + + +`, + ], + desc: ' - Powered by RSSHub', + }, + partlyRelayed: { + items: [ + ` + + + +`, + ], + desc: ' - Powered by RSSHub', + }, + }, + extraComplicated: { + origin: { + items: [ + { + content: + '\n\n\n\n\n\n\n\n\n\n', + itunes: {}, + }, + { + content: '\n', + itunes: {}, + }, + { + content: + '\n\n', + enclosure: { + url: 'https://mock.com/DIYgod/RSSHub.png', + type: 'image/png', + }, + itunes: { + image: 'https://mock.com/DIYgod/RSSHub.gif', + }, + }, + ], + image: { + link: 'https://github.com/DIYgod/RSSHub', + url: 'https://mock.com/DIYgod/RSSHub.png', + title: 'Test complicated', + }, + description: ' - Powered by RSSHub', + }, + processed: { + items: [ + { + content: + '\n\n\n\n\n\n\n\n\n\n', + itunes: {}, + }, + { + content: '\n', + itunes: {}, + }, + { + content: + '\n\n', + enclosure: { + url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.png', + type: 'image/png', + }, + itunes: { + image: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.gif', + }, + }, + ], + image: { + link: 'https://github.com/DIYgod/RSSHub', + url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.png', + title: 'Test complicated', + }, + description: ' - Powered by RSSHub', + }, + urlencoded: { + items: [ + { + content: + '\n\n\n\n\n\n\n\n\n\n', + itunes: {}, + }, + { + content: '\n', + itunes: {}, + }, + { + content: + '\n\n', + enclosure: { + url: 'https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.png', + type: 'image/png', + }, + itunes: { + image: 'https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.gif', + }, + }, + ], + image: { + link: 'https://github.com/DIYgod/RSSHub', + url: 'https://images.weserv.nl?url=https%3A%2F%2Fmock.com%2FDIYgod%2FRSSHub.png', + title: 'Test complicated', + }, + description: ' - Powered by RSSHub', + }, + }, + extraMultimedia: { + origin: { + items: [ + { + content: + '\n\n\n\n', + }, + { + content: '\n', + enclosure: { + url: 'https://mock.com/DIYgod/RSSHub.mp4', + type: 'video/mp4', + }, + }, + ], + description: ' - Powered by RSSHub', + }, + relayed: { + items: [ + { + content: + '\n\n\n\n', + }, + { + content: '\n', + enclosure: { + url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4', + type: 'video/mp4', + }, + }, + ], + description: ' - Powered by RSSHub', + }, + partlyRelayed: { + items: [ + { + content: + '\n\n\n\n', + }, + { + content: '\n', + enclosure: { + url: 'https://i3.wp.com/mock.com/DIYgod/RSSHub.mp4', + type: 'video/mp4', + }, + }, + ], + description: ' - Powered by RSSHub', + }, + }, +}; + +const testAntiHotlink = async (path, expectObj, query?: string | Record) => { + const app = (await import('@/app')).default; + + let queryStr; + if (query) { + queryStr = + typeof query === 'string' + ? query + : Object.entries(query) + .map(([key, value]) => `${key}=${value}`) + .join('&'); + } + path = path + (queryStr ? `?${queryStr}` : ''); + + const response = await app.request(path); + const parsed = await parser.parseString(await response.text()); + expect({ + items: parsed.items.slice(0, expectObj.items.length).map((i) => i.content), + desc: parsed.description, + }).toStrictEqual(expectObj); + + return parsed; +}; + +const testAntiHotlinkExtra = async (path, expectObj, query?: string | Record) => { + const app = (await import('@/app')).default; + + path += query ? `?${new URLSearchParams(query).toString()}` : ''; + + const response = await app.request(path); + const parsed = await parser.parseString(await response.text()); + const obj = { + description: parsed.description, + image: parsed.image, + items: parsed.items.slice(0, expectObj.items.length).map((e) => ({ + content: e.content, + enclosure: e.enclosure, + itunes: e.itunes, + })), + }; + expect(obj).toEqual(expectObj); + + return parsed; +}; + +const expectImgOrigin = async (query?: string | Record) => { + await testAntiHotlink('/test/complicated', expects.complicated.origin, query); + await testAntiHotlinkExtra('/test/complicated', expects.extraComplicated.origin, query); +}; +const expectImgProcessed = async (query?: string | Record) => { + await testAntiHotlink('/test/complicated', expects.complicated.processed, query); + await testAntiHotlinkExtra('/test/complicated', expects.extraComplicated.processed, query); +}; + +const expectImgUrlencoded = async (query?: string | Record) => { + await testAntiHotlink('/test/complicated', expects.complicated.urlencoded, query); + await testAntiHotlinkExtra('/test/complicated', expects.extraComplicated.urlencoded, query); +}; + +const expectMultimediaOrigin = async (query?: string | Record) => { + await testAntiHotlink('/test/multimedia', expects.multimedia.origin, query); + await testAntiHotlinkExtra('/test/multimedia', expects.extraMultimedia.origin, query); +}; + +const expectMultimediaRelayed = async (query?: string | Record) => { + await testAntiHotlink('/test/multimedia', expects.multimedia.relayed, query); + await testAntiHotlinkExtra('/test/multimedia', expects.extraMultimedia.relayed, query); +}; + +const expectMultimediaPartlyRelayed = async (query?: string | Record) => { + await testAntiHotlink('/test/multimedia', expects.multimedia.partlyRelayed, query); + await testAntiHotlinkExtra('/test/multimedia', expects.extraMultimedia.partlyRelayed, query); +}; + +describe('anti-hotlink', () => { + it('template-legacy', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + await expectImgProcessed(); + }); + + it('template-experimental', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.ALLOW_USER_HOTLINK_TEMPLATE = 'true'; + await expectImgProcessed(); + await expectMultimediaRelayed({ multimedia_hotlink_template: process.env.HOTLINK_TEMPLATE }); + }); + + it('url', async () => { + process.env.HOTLINK_TEMPLATE = '${protocol}//${host}${pathname}'; + await expectImgOrigin(); + await expectMultimediaOrigin({ multimedia_hotlink_template: process.env.HOTLINK_TEMPLATE }); + }); + + it('url-encoded', async () => { + process.env.HOTLINK_TEMPLATE = 'https://images.weserv.nl?url=${href_ue}'; + await expectImgUrlencoded(); + }); + + it('template-priority-legacy', async () => { + process.env.HOTLINK_TEMPLATE = '${protocol}//${host}${pathname}'; + await expectImgOrigin(); + }); + + it('template-priority-experimental', async () => { + process.env.ALLOW_USER_HOTLINK_TEMPLATE = 'true'; + await expectImgOrigin(); + await expectImgProcessed({ image_hotlink_template: 'https://i3.wp.com/${host}${pathname}' }); + }); + + it('no-template', async () => { + process.env.HOTLINK_TEMPLATE = ''; + await expectImgOrigin(); + await expectMultimediaOrigin(); + }); + + it('multimedia-template-experimental', async () => { + process.env.ALLOW_USER_HOTLINK_TEMPLATE = 'true'; + await expectMultimediaOrigin({ multimedia_hotlink_template: '${protocol}//${host}${pathname}' }); + await expectMultimediaPartlyRelayed({ multimedia_hotlink_template: 'https://i3.wp.com/${host}${pathname}' }); + }); + + it('include-paths-partial-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test'; + await expectImgProcessed(); + }); + + it('include-paths-fully-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test/complicated'; + await expectImgProcessed(); + }); + + it('include-paths-unmatched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/t'; + await expectImgOrigin(); + }); + + it('exclude-paths-partial-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test'; + await expectImgOrigin(); + }); + + it('exclude-paths-fully-matched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test/complicated'; + await expectImgOrigin(); + }); + + it('exclude-paths-unmatched', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_EXCLUDE_PATHS = '/t'; + await expectImgProcessed(); + }); + + it('include-exclude-paths-mixed-filtered-out', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test/complicated'; + await expectImgOrigin(); + }); + + it('include-exclude-paths-mixed-unfiltered-out', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${host}${pathname}'; + process.env.HOTLINK_INCLUDE_PATHS = '/test'; + process.env.HOTLINK_EXCLUDE_PATHS = '/test/c'; + await expectImgProcessed(); + }); + + it('invalid-property', async () => { + process.env.HOTLINK_TEMPLATE = 'https://i3.wp.com/${createObjectURL}'; + const app = (await import('@/app')).default; + const response = await app.request('/test/complicated'); + expect(await response.text()).toContain('Error: Invalid URL property: createObjectURL'); + }); +}); diff --git a/lib/middleware/anti-hotlink.ts b/lib/middleware/anti-hotlink.ts new file mode 100644 index 00000000000000..6b04cda304c769 --- /dev/null +++ b/lib/middleware/anti-hotlink.ts @@ -0,0 +1,168 @@ +import { config } from '@/config'; +import { load, type CheerioAPI } from 'cheerio'; +import logger from '@/utils/logger'; +import { type MiddlewareHandler } from 'hono'; +import { Data } from '@/types'; + +const templateRegex = /\${([^{}]+)}/g; +const allowedUrlProperties = new Set(['hash', 'host', 'hostname', 'href', 'origin', 'password', 'pathname', 'port', 'protocol', 'search', 'searchParams', 'username']); + +// match path or sub-path +const matchPath = (path: string, paths: string[]) => { + for (const p of paths) { + if (path.startsWith(p) && (path.length === p.length || path[p.length] === '/')) { + return true; + } + } + return false; +}; + +// return true if the path needs to be processed +const filterPath = (path: string) => { + const include = config.hotlink.includePaths; + const exclude = config.hotlink.excludePaths; + return !(include && !matchPath(path, include)) && !(exclude && matchPath(path, exclude)); +}; + +const interpolate = (str: string, obj: Record) => + str.replaceAll(templateRegex, (_, prop) => { + let needEncode = false; + if (prop.endsWith('_ue')) { + // url encode + prop = prop.slice(0, -3); + needEncode = true; + } + return needEncode ? encodeURIComponent(obj[prop]) : obj[prop]; + }); +const parseUrl = (str: string) => { + let url; + try { + url = new URL(str); + } catch { + logger.error(`Failed to parse ${str}`); + } + + return url; +}; + +const replaceUrl = (template?: string, url?: string) => { + if (!template || !url) { + return url; + } + const oldUrl = parseUrl(url); + if (oldUrl && oldUrl.protocol !== 'data:') { + return interpolate(template, oldUrl); + } + return url; +}; + +const replaceUrls = ($: CheerioAPI, selector: string, template: string, attribute = 'src') => { + $(selector).each(function () { + const oldSrc = $(this).attr(attribute); + if (oldSrc) { + const url = parseUrl(oldSrc); + if (url && url.protocol !== 'data:') { + // Cheerio will do the right thing to prohibit XSS. + $(this).attr(attribute, interpolate(template, url)); + } + } + }); +}; + +const process = (html: string, image_hotlink_template?: string, multimedia_hotlink_template?: string) => { + const $ = load(html, undefined, false); + if (image_hotlink_template) { + replaceUrls($, 'img, picture > source', image_hotlink_template); + replaceUrls($, 'video[poster]', image_hotlink_template, 'poster'); + replaceUrls($, '*[data-rsshub-image="href"]', image_hotlink_template, 'href'); + } + if (multimedia_hotlink_template) { + replaceUrls($, 'video, video > source, audio, audio > source', multimedia_hotlink_template); + if (!image_hotlink_template) { + replaceUrls($, 'video[poster]', multimedia_hotlink_template, 'poster'); + } + } + return $.html(); +}; + +const validateTemplate = (template?: string) => { + if (!template) { + return; + } + for (const match of template.matchAll(templateRegex)) { + const prop = match[1].endsWith('_ue') ? match[1].slice(0, -3) : match[1]; + if (!allowedUrlProperties.has(prop)) { + throw new Error(`Invalid URL property: ${prop}`); + } + } +}; + +const middleware: MiddlewareHandler = async (ctx, next) => { + await next(); + + let imageHotlinkTemplate: string | undefined; + let multimediaHotlinkTemplate: string | undefined; + + // Read params if enabled + if (config.feature.allow_user_hotlink_template) { + // By default, the config turns these features off. Set corresponding config to + // true to turn this feature on. + // A risk is that the media URLs will be replaced by user-supplied templates, + // so a user could literally take the control of "where are the media from", + // but only in their personal-use feed URL. + multimediaHotlinkTemplate = ctx.req.query('multimedia_hotlink_template'); + imageHotlinkTemplate = ctx.req.query('image_hotlink_template'); + } + + // Force config hotlink template on conflict + if (config.hotlink.template) { + imageHotlinkTemplate = filterPath(ctx.req.path) ? config.hotlink.template : undefined; + multimediaHotlinkTemplate = filterPath(ctx.req.path) ? config.hotlink.template : undefined; + } + + if (!imageHotlinkTemplate && !multimediaHotlinkTemplate) { + return; + } + + validateTemplate(imageHotlinkTemplate); + validateTemplate(multimediaHotlinkTemplate); + + // Assume that only description include image link + // and here we will only check them in description. + // Use Cheerio to load the description as html and filter all + // image link + const data: Data = ctx.get('data'); + if (data) { + if (data.image) { + data.image = replaceUrl(imageHotlinkTemplate, data.image); + } + if (data.description) { + data.description = process(data.description, imageHotlinkTemplate, multimediaHotlinkTemplate); + } + + if (data.item) { + for (const item of data.item) { + if (item.description) { + item.description = process(item.description, imageHotlinkTemplate, multimediaHotlinkTemplate); + } + if (item.enclosure_url && item.enclosure_type) { + if (item.enclosure_type.startsWith('image/')) { + item.enclosure_url = replaceUrl(imageHotlinkTemplate, item.enclosure_url); + } else if (/^(video|audio)\//.test(item.enclosure_type)) { + item.enclosure_url = replaceUrl(multimediaHotlinkTemplate, item.enclosure_url); + } + } + if (item.image) { + item.image = replaceUrl(imageHotlinkTemplate, item.image); + } + if (item.itunes_item_image) { + item.itunes_item_image = replaceUrl(imageHotlinkTemplate, item.itunes_item_image); + } + } + } + + ctx.set('data', data); + } +}; + +export default middleware; diff --git a/lib/middleware/api-response-handler.js b/lib/middleware/api-response-handler.js deleted file mode 100644 index e43954750c4c49..00000000000000 --- a/lib/middleware/api-response-handler.js +++ /dev/null @@ -1,147 +0,0 @@ -/** - * HTTP Status codes - */ -const statusCodes = { - CONTINUE: 100, - OK: 200, - CREATED: 201, - ACCEPTED: 202, - NO_CONTENT: 204, - BAD_REQUEST: 400, - UNAUTHORIZED: 401, - FORBIDDEN: 403, - NOT_FOUND: 404, - REQUEST_TIMEOUT: 408, - UNPROCESSABLE_ENTITY: 422, - INTERNAL_SERVER_ERROR: 500, - NOT_IMPLEMENTED: 501, - BAD_GATEWAY: 502, - SERVICE_UNAVAILABLE: 503, - GATEWAY_TIME_OUT: 504, -}; - -function responseHandler() { - return async (ctx, next) => { - ctx.res.statusCodes = statusCodes; - ctx.statusCodes = ctx.res.statusCodes; - - ctx.res.success = ({ statusCode, data = null, message = null }) => { - const status = 0; - - ctx.status = statusCode; - ctx.body = { status, data, message }; - }; - - // ctx.res.fail = ({ statusCode, code, data = null, message = null }) => { - // const status = -1; - - // if (!!statusCode && (statusCode >= 400 && statusCode < 500)) { - // ctx.status = statusCode; - // } else if (!(ctx.status >= 400 && ctx.status < 500)) { - // ctx.status = statusCodes.BAD_REQUEST; - // } - - // ctx.body = { status, code, data, message }; - // }; - - // ctx.res.error = ({ statusCode, code, data = null, message = null }) => { - // const status = -2; - - // if (!!statusCode && (statusCode >= 500 && statusCode < 600)) { - // ctx.status = statusCode; - // } else if (!(ctx.status >= 500 && ctx.status < 600)) { - // ctx.status = statusCodes.INTERNAL_SERVER_ERROR; - // } - - // ctx.body = { status, code, data, message }; - // }; - - ctx.res.ok = (params = {}) => { - ctx.res.success({ - ...params, - statusCode: statusCodes.OK, - }); - }; - - // ctx.res.noContent = (params = {}) => { - // ctx.res.success({ - // ...params, - // statusCode: statusCodes.NO_CONTENT, - // }); - // }; - - // ctx.res.badRequest = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.BAD_REQUEST, - // }); - // }; - - // ctx.res.forbidden = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.FORBIDDEN, - // }); - // }; - - // ctx.res.notFound = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.NOT_FOUND, - // }); - // }; - - // ctx.res.requestTimeout = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.REQUEST_TIMEOUT, - // }); - // }; - - // ctx.res.unprocessableEntity = (params = {}) => { - // ctx.res.fail({ - // ...params, - // statusCode: statusCodes.UNPROCESSABLE_ENTITY, - // }); - // }; - - // ctx.res.internalServerError = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.INTERNAL_SERVER_ERROR, - // }); - // }; - - // ctx.res.notImplemented = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.NOT_IMPLEMENTED, - // }); - // }; - - // ctx.res.badGateway = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.BAD_GATEWAY, - // }); - // }; - - // ctx.res.serviceUnavailable = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.SERVICE_UNAVAILABLE, - // }); - // }; - - // ctx.res.gatewayTimeOut = (params = {}) => { - // ctx.res.error({ - // ...params, - // statusCode: statusCodes.GATEWAY_TIME_OUT, - // }); - // }; - - await next(); - }; -} - -module.exports = responseHandler; diff --git a/lib/middleware/api-template.js b/lib/middleware/api-template.js deleted file mode 100644 index c3f10706db6708..00000000000000 --- a/lib/middleware/api-template.js +++ /dev/null @@ -1,9 +0,0 @@ -module.exports = async (ctx, next) => { - await next(); - if (ctx.request.path.startsWith('/api/')) { - return ctx.res.ok({ - message: `request returned ${ctx.body.counter} ${ctx.body.counter > 1 ? 'routes' : 'route'}`, - data: ctx.body.result, - }); - } -}; diff --git a/lib/middleware/cache.test.ts b/lib/middleware/cache.test.ts new file mode 100644 index 00000000000000..703da060cd405e --- /dev/null +++ b/lib/middleware/cache.test.ts @@ -0,0 +1,189 @@ +import { describe, expect, it, vi, afterEach } from 'vitest'; +import Parser from 'rss-parser'; +import wait from '@/utils/wait'; + +process.env.CACHE_EXPIRE = '1'; +process.env.CACHE_CONTENT_EXPIRE = '2'; + +const parser = new Parser(); + +afterEach(() => { + vi.resetModules(); +}); + +const noCacheTestFunc = async () => { + const app = (await import('@/app')).default; + + const response1 = await app.request('/test/cache'); + const response2 = await app.request('/test/cache'); + + const parsed1 = await parser.parseString(await response1.text()); + const parsed2 = await parser.parseString(await response2.text()); + + expect(response2.status).toBe(200); + expect(response2.headers).not.toHaveProperty('rsshub-cache-status'); + + expect(parsed1.items[0].content).toBe('Cache1'); + expect(parsed2.items[0].content).toBe('Cache2'); + + expect(parsed1.ttl).toEqual('1'); +}; + +describe('cache', () => { + it('memory', async () => { + process.env.CACHE_TYPE = 'memory'; + const app = (await import('@/app')).default; + + const response1 = await app.request('/test/cache'); + const response2 = await app.request('/test/cache'); + + const parsed1 = await parser.parseString(await response1.text()); + const parsed2 = await parser.parseString(await response2.text()); + + delete parsed1.lastBuildDate; + delete parsed2.lastBuildDate; + delete parsed1.feedUrl; + delete parsed2.feedUrl; + delete parsed1.paginationLinks; + delete parsed2.paginationLinks; + expect(parsed2).toMatchObject(parsed1); + + expect(response2.status).toBe(200); + expect(response2.headers.get('rsshub-cache-status')).toBe('HIT'); + + expect(parsed1.ttl).toEqual('1'); + + await wait(1 * 1000 + 100); + const response3 = await app.request('/test/cache'); + expect(response3.headers).not.toHaveProperty('rsshub-cache-status'); + const parsed3 = await parser.parseString(await response3.text()); + + await wait(2 * 1000 + 100); + const response4 = await app.request('/test/cache'); + const parsed4 = await parser.parseString(await response4.text()); + + expect(parsed1.items[0].content).toBe('Cache1'); + expect(parsed2.items[0].content).toBe('Cache1'); + expect(parsed3.items[0].content).toBe('Cache1'); + expect(parsed4.items[0].content).toBe('Cache2'); + + await app.request('/test/refreshCache'); + await wait(1 * 1000 + 100); + const response5 = await app.request('/test/refreshCache'); + const parsed5 = await parser.parseString(await response5.text()); + await wait(1 * 1000 + 100); + const response6 = await app.request('/test/refreshCache'); + const parsed6 = await parser.parseString(await response6.text()); + + expect(parsed5.items[0].content).toBe('1 1'); + expect(parsed6.items[0].content).toBe('1 0'); + }, 10000); + + it('redis', async () => { + process.env.CACHE_TYPE = 'redis'; + const app = (await import('@/app')).default; + + await wait(500); + const response1 = await app.request('/test/cache'); + const response2 = await app.request('/test/cache'); + + const parsed1 = await parser.parseString(await response1.text()); + const parsed2 = await parser.parseString(await response2.text()); + + delete parsed1.lastBuildDate; + delete parsed2.lastBuildDate; + delete parsed1.feedUrl; + delete parsed2.feedUrl; + delete parsed1.paginationLinks; + delete parsed2.paginationLinks; + expect(parsed2).toMatchObject(parsed1); + + expect(response2.status).toBe(200); + expect(response2.headers.get('rsshub-cache-status')).toBe('HIT'); + + expect(parsed1.ttl).toEqual('1'); + + await wait(1 * 1000 + 100); + const response3 = await app.request('/test/cache'); + expect(response3.headers).not.toHaveProperty('rsshub-cache-status'); + const parsed3 = await parser.parseString(await response3.text()); + + await wait(2 * 1000 + 100); + const response4 = await app.request('/test/cache'); + const parsed4 = await parser.parseString(await response4.text()); + + expect(parsed1.items[0].content).toBe('Cache1'); + expect(parsed2.items[0].content).toBe('Cache1'); + expect(parsed3.items[0].content).toBe('Cache1'); + expect(parsed4.items[0].content).toBe('Cache2'); + + await app.request('/test/refreshCache'); + await wait(1 * 1000 + 100); + const response5 = await app.request('/test/refreshCache'); + const parsed5 = await parser.parseString(await response5.text()); + await wait(1 * 1000 + 100); + const response6 = await app.request('/test/refreshCache'); + const parsed6 = await parser.parseString(await response6.text()); + + expect(parsed5.items[0].content).toBe('1 1'); + expect(parsed6.items[0].content).toBe('1 0'); + + const cache = (await import('@/utils/cache')).default; + await cache.clients.redisClient!.quit(); + }, 10000); + + it('redis with quit', async () => { + process.env.CACHE_TYPE = 'redis'; + const cache = (await import('@/utils/cache')).default; + await cache.clients.redisClient!.quit(); + await noCacheTestFunc(); + }); + + it('redis with error', async () => { + process.env.CACHE_TYPE = 'redis'; + process.env.REDIS_URL = 'redis://wrongpath:6379'; + await noCacheTestFunc(); + const cache = (await import('@/utils/cache')).default; + await cache.clients.redisClient!.quit(); + }); + + it('no cache', async () => { + process.env.CACHE_TYPE = 'NO'; + await noCacheTestFunc(); + }); + + it('no cache (empty string)', async () => { + process.env.CACHE_TYPE = ''; + await noCacheTestFunc(); + }); + + it('throws URL key', async () => { + process.env.CACHE_TYPE = 'memory'; + const app = (await import('@/app')).default; + + try { + const response = await app.request('/test/cacheUrlKey'); + expect(response).toThrow(Error); + } catch (error: any) { + expect(error.message).toContain('Cache key must be a string'); + } + }); + + it('RSS TTL (no cache)', async () => { + process.env.CACHE_TYPE = ''; + process.env.CACHE_EXPIRE = '600'; + const app = (await import('@/app')).default; + const response = await app.request('/test/cache'); + const parsed = await parser.parseString(await response.text()); + expect(parsed.ttl).toEqual('1'); + }); + + it('RSS TTL (w/ cache)', async () => { + process.env.CACHE_TYPE = 'memory'; + process.env.CACHE_EXPIRE = '600'; + const app = (await import('@/app')).default; + const response = await app.request('/test/cache'); + const parsed = await parser.parseString(await response.text()); + expect(parsed.ttl).toEqual('10'); + }); +}); diff --git a/lib/middleware/cache.ts b/lib/middleware/cache.ts new file mode 100644 index 00000000000000..d31a26a5b0b13b --- /dev/null +++ b/lib/middleware/cache.ts @@ -0,0 +1,78 @@ +import xxhash from 'xxhash-wasm'; +import type { MiddlewareHandler } from 'hono'; + +import { config } from '@/config'; +import RequestInProgressError from '@/errors/types/request-in-progress'; +import cacheModule from '@/utils/cache/index'; +import { Data } from '@/types'; + +const bypassList = new Set(['/', '/robots.txt', '/logo.png', '/favicon.ico']); +// only give cache string, as the `!` condition tricky +// XXH64 is used to shrink key size +// plz, write these tips in comments! +const middleware: MiddlewareHandler = async (ctx, next) => { + if (!cacheModule.status.available || bypassList.has(ctx.req.path)) { + await next(); + return; + } + + const requestPath = ctx.req.path; + const limit = ctx.req.query('limit') ? `:${ctx.req.query('limit')}` : ''; + const { h64ToString } = await xxhash(); + const key = 'rsshub:koa-redis-cache:' + h64ToString(requestPath + limit); + const controlKey = 'rsshub:path-requested:' + h64ToString(requestPath + limit); + + const isRequesting = await cacheModule.globalCache.get(controlKey); + + if (isRequesting === '1') { + let retryTimes = process.env.NODE_ENV === 'test' ? 1 : 10; + let bypass = false; + while (retryTimes > 0) { + // eslint-disable-next-line no-await-in-loop + await new Promise((resolve) => setTimeout(resolve, process.env.NODE_ENV === 'test' ? 3000 : 6000)); + // eslint-disable-next-line no-await-in-loop + if ((await cacheModule.globalCache.get(controlKey)) !== '1') { + bypass = true; + break; + } + retryTimes--; + } + if (!bypass) { + throw new RequestInProgressError('This path is currently fetching, please come back later!'); + } + } + + const value = await cacheModule.globalCache.get(key); + + if (value) { + ctx.status(200); + ctx.header('RSSHub-Cache-Status', 'HIT'); + ctx.set('data', JSON.parse(value)); + await next(); + return; + } + + // Doesn't hit the cache? We need to let others know! + await cacheModule.globalCache.set(controlKey, '1', config.cache.requestTimeout); + + try { + await next(); + } catch (error) { + await cacheModule.globalCache.set(controlKey, '0', config.cache.requestTimeout); + throw error; + } + + const data: Data = ctx.get('data'); + if (ctx.res.headers.get('Cache-Control') !== 'no-cache' && data) { + data.lastBuildDate = new Date().toUTCString(); + ctx.set('data', data); + const body = JSON.stringify(data); + await cacheModule.globalCache.set(key, body, config.cache.routeExpire); + } + + // We need to let it go, even no cache set. + // Wait to set cache so the next request could be handled correctly + await cacheModule.globalCache.set(controlKey, '0', config.cache.requestTimeout); +}; + +export default middleware; diff --git a/lib/middleware/cache/index.js b/lib/middleware/cache/index.js deleted file mode 100644 index 52c0b3aeaaec0f..00000000000000 --- a/lib/middleware/cache/index.js +++ /dev/null @@ -1,138 +0,0 @@ -const md5 = require('@/utils/md5'); -const config = require('@/config').value; -const logger = require('@/utils/logger'); -const { RequestInProgressError } = require('@/errors'); - -const globalCache = { - get: () => null, - set: () => null, -}; - -let cacheModule = { - get: () => null, - set: () => null, - status: { available: false }, - clients: {}, -}; - -if (config.cache.type === 'redis') { - cacheModule = require('./redis'); - const { redisClient } = cacheModule.clients; - globalCache.get = async (key) => { - if (key && cacheModule.status.available) { - const value = await redisClient.get(key); - return value; - } - }; - globalCache.set = cacheModule.set; -} else if (config.cache.type === 'memory') { - cacheModule = require('./memory'); - const { memoryCache } = cacheModule.clients; - globalCache.get = (key) => { - if (key && cacheModule.status.available) { - return memoryCache.get(key, { updateAgeOnGet: false }); - } - }; - globalCache.set = (key, value, maxAge) => { - if (!value || value === 'undefined') { - value = ''; - } - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (key) { - return memoryCache.set(key, value, { ttl: maxAge * 1000 }); - } - }; -} else { - logger.error('Cache not available, concurrent requests are not limited. This could lead to bad behavior.'); -} - -// only give cache string, as the `!` condition tricky -// md5 is used to shrink key size -// plz, write these tips in comments! -module.exports = function (app) { - const { get, set, status } = cacheModule; - app.context.cache = { - ...cacheModule, - tryGet: async (key, getValueFunc, maxAge = config.cache.contentExpire, refresh = true) => { - if (typeof key !== 'string') { - throw Error('Cache key must be a string'); - } - let v = await get(key, refresh); - if (!v) { - v = await getValueFunc(); - set(key, v, maxAge); - } else { - let parsed; - try { - parsed = JSON.parse(v); - } catch (e) { - parsed = null; - } - if (parsed) { - v = parsed; - } - } - - return v; - }, - globalCache, - }; - - return async (ctx, next) => { - const key = 'koa-redis-cache:' + md5(ctx.request.path); - const controlKey = 'path-requested:' + md5(ctx.request.path); - - if (!status.available) { - return next(); - } - - const isRequesting = await globalCache.get(controlKey); - - if (isRequesting === '1') { - throw new RequestInProgressError('This path is currently fetching, please come back later!'); - } - - try { - const value = await globalCache.get(key); - - if (value) { - ctx.response.status = 200; - if (config.cache.type === 'redis') { - ctx.response.set({ - 'X-Koa-Redis-Cache': 'true', - }); - } else if (config.cache.type === 'memory') { - ctx.response.set({ - 'X-Koa-Memory-Cache': 'true', - }); - } - ctx.state.data = JSON.parse(value); - return; - } - } catch (e) { - // - } - - // Doesn't hit the cache? We need to let others know! - await globalCache.set(controlKey, '1', config.cache.requestTimeout); - - try { - await next(); - } catch (e) { - await globalCache.set(controlKey, '0', config.cache.requestTimeout); - throw e; - } - - if (ctx.response.get('Cache-Control') !== 'no-cache' && ctx.state && ctx.state.data) { - ctx.state.data.lastBuildDate = new Date().toUTCString(); - const body = JSON.stringify(ctx.state.data); - await globalCache.set(key, body, config.cache.routeExpire); - } - - // We need to let it go, even no cache set. - // Wait to set cache so the next request could be handled correctly - await globalCache.set(controlKey, '0', config.cache.requestTimeout); - }; -}; diff --git a/lib/middleware/cache/memory.js b/lib/middleware/cache/memory.js deleted file mode 100644 index 21dc9c7d41b253..00000000000000 --- a/lib/middleware/cache/memory.js +++ /dev/null @@ -1,36 +0,0 @@ -const { LRUCache } = require('lru-cache'); -const config = require('@/config').value; - -const status = { available: false }; - -const memoryCache = new LRUCache({ - ttl: config.cache.routeExpire * 1000, - max: config.memory.max, -}); - -status.available = true; - -module.exports = { - get: (key, refresh = true) => { - if (key && status.available) { - let value = memoryCache.get(key, { updateAgeOnGet: refresh }); - if (value) { - value = value + ''; - } - return value; - } - }, - set: (key, value, maxAge = config.cache.contentExpire) => { - if (!value || value === 'undefined') { - value = ''; - } - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (key && status.available) { - return memoryCache.set(key, value, { ttl: maxAge * 1000 }); - } - }, - clients: { memoryCache }, - status, -}; diff --git a/lib/middleware/cache/redis.js b/lib/middleware/cache/redis.js deleted file mode 100644 index 1e9889d0f6f7f9..00000000000000 --- a/lib/middleware/cache/redis.js +++ /dev/null @@ -1,68 +0,0 @@ -const config = require('@/config').value; -const Redis = require('ioredis'); -const logger = require('@/utils/logger'); - -const redisClient = new Redis(config.redis.url); - -const status = { available: false }; - -redisClient.on('error', (error) => { - status.available = false; - logger.error('Redis error: ', error); -}); -redisClient.on('end', () => { - status.available = false; -}); -redisClient.on('connect', () => { - status.available = true; - logger.info('Redis connected.'); -}); - -const getCacheTtlKey = (key) => { - if (key.startsWith('cacheTtl:')) { - throw Error('"cacheTtl:" prefix is reserved for the internal usage, please change your cache key'); // blocking any attempt to get/set the cacheTtl - } - return `cacheTtl:${key}`; -}; - -module.exports = { - get: async (key, refresh = true) => { - if (key && status.available) { - const cacheTtlKey = getCacheTtlKey(key); - let [value, cacheTtl] = await redisClient.mget(key, cacheTtlKey); - if (value && refresh) { - if (!cacheTtl) { - // if cacheTtl is not set, that means the cache expire time is contentExpire - cacheTtl = config.cache.contentExpire; - // dont save cacheTtl to Redis, as it is the default value - // redisClient.set(cacheTtlKey, cacheTtl, 'EX', cacheTtl); - } else { - redisClient.expire(cacheTtlKey, cacheTtl); - } - redisClient.expire(key, cacheTtl); - value = value + ''; - } - return value; - } - }, - set: (key, value, maxAge = config.cache.contentExpire) => { - if (!status.available) { - return; - } - if (!value || value === 'undefined') { - value = ''; - } - if (typeof value === 'object') { - value = JSON.stringify(value); - } - if (key) { - if (maxAge !== config.cache.contentExpire) { - // Only set cacheTtlKey if maxAge !== contentExpire - redisClient.set(getCacheTtlKey(key), maxAge, 'EX', maxAge); - } - return redisClient.set(key, value, 'EX', maxAge); // setMode: https://redis.io/commands/set - } - }, - clients: { redisClient }, - status, -}; diff --git a/lib/middleware/debug.js b/lib/middleware/debug.js deleted file mode 100644 index 644795418aae91..00000000000000 --- a/lib/middleware/debug.js +++ /dev/null @@ -1,25 +0,0 @@ -module.exports = async (ctx, next) => { - if (!ctx.debug.paths[ctx.request.path]) { - ctx.debug.paths[ctx.request.path] = 0; - } - ctx.debug.paths[ctx.request.path]++; - - ctx.debug.request++; - - await next(); - - if (!ctx.debug.routes[ctx._matchedRoute]) { - ctx._matchedRoute && (ctx.debug.routes[ctx._matchedRoute] = 0); - } - ctx._matchedRoute && ctx.debug.routes[ctx._matchedRoute]++; - - if (ctx.response.get('X-Koa-Redis-Cache') || ctx.response.get('X-Koa-Memory-Cache')) { - ctx.debug.hitCache++; - } - - ctx.state.debuged = true; - - if (ctx.status === 304) { - ctx.debug.etag++; - } -}; diff --git a/lib/middleware/debug.test.ts b/lib/middleware/debug.test.ts new file mode 100644 index 00000000000000..7bcf980e8c45cc --- /dev/null +++ b/lib/middleware/debug.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, it } from 'vitest'; +import app from '@/app'; +import { load } from 'cheerio'; + +process.env.NODE_NAME = 'mock'; + +describe('debug', () => { + it('debug', async () => { + const response1 = await app.request('/test/1'); + const etag = response1.headers.get('etag'); + await app.request('/test/1', { + headers: { + 'If-None-Match': etag!, + }, + }); + await app.request('/test/2'); + await app.request('/test/empty'); + await app.request('/test/empty'); + + const response = await app.request('/'); + + const $ = load(await response.text()); + $('.debug-item').each((index, item) => { + const key = $(item).find('.debug-key').html()?.trim(); + const value = $(item).find('.debug-value').html()?.trim(); + switch (key) { + case 'Node Name:': + expect(value).toBe('mock'); + break; + case 'Request Amount:': + expect(value).toBe('6'); + break; + case 'ETag Matched:': + expect(value).toBe('1'); + break; + default: + } + }); + }); +}); diff --git a/lib/middleware/debug.ts b/lib/middleware/debug.ts new file mode 100644 index 00000000000000..c9fc8799f55997 --- /dev/null +++ b/lib/middleware/debug.ts @@ -0,0 +1,37 @@ +import { MiddlewareHandler } from 'hono'; +import { getDebugInfo, setDebugInfo } from '@/utils/debug-info'; + +const middleware: MiddlewareHandler = async (ctx, next) => { + { + const debug = getDebugInfo(); + if (!debug.paths[ctx.req.path]) { + debug.paths[ctx.req.path] = 0; + } + debug.paths[ctx.req.path]++; + + debug.request++; + setDebugInfo(debug); + } + + await next(); + + { + const debug = getDebugInfo(); + const hasMatchedRoute = ctx.req.routePath !== '/*'; + if (!debug.routes[ctx.req.routePath] && hasMatchedRoute) { + debug.routes[ctx.req.routePath] = 0; + } + hasMatchedRoute && debug.routes[ctx.req.routePath]++; + + if (ctx.res.headers.get('RSSHub-Cache-Status')) { + debug.hitCache++; + } + + if (ctx.res.status === 304) { + debug.etag++; + } + setDebugInfo(debug); + } +}; + +export default middleware; diff --git a/lib/middleware/filter-engine.test.ts b/lib/middleware/filter-engine.test.ts new file mode 100644 index 00000000000000..1a9d6e60b6a90c --- /dev/null +++ b/lib/middleware/filter-engine.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it, afterAll, vi, afterEach } from 'vitest'; + +afterAll(() => { + delete process.env.FILTER_REGEX_ENGINE; +}); + +afterEach(() => { + delete process.env.FILTER_REGEX_ENGINE; + vi.resetModules(); +}); + +describe('filter-engine', () => { + it(`filter RE2 engine ReDoS attack`, async () => { + const app = (await import('@/app')).default; + + const response = await app.request('/test/1?filter=abc(%3F%3Ddef)'); + expect(response.status).toBe(503); + expect(await response.text()).toMatch(/RE2JSSyntaxException/); + }); + + it(`filter Regexp engine backward compatibility`, async () => { + process.env.FILTER_REGEX_ENGINE = 'regexp'; + + const app = (await import('@/app')).default; + + const response = await app.request('/test/1?filter=abc(%3F%3Ddef)'); + expect(response.status).toBe(200); + }); + + it(`filter Regexp engine test config`, async () => { + process.env.FILTER_REGEX_ENGINE = 'somethingelse'; + + const app = (await import('@/app')).default; + + const response = await app.request('/test/1?filter=abc(%3F%3Ddef)'); + expect(response.status).toBe(503); + expect(await response.text()).toMatch(/somethingelse/); + }); +}); diff --git a/lib/middleware/header.js b/lib/middleware/header.js deleted file mode 100644 index 2292ddbed4eeac..00000000000000 --- a/lib/middleware/header.js +++ /dev/null @@ -1,45 +0,0 @@ -const etagCalculate = require('etag'); -const logger = require('@/utils/logger'); -const config = require('@/config').value; -const headers = { - 'Access-Control-Allow-Methods': 'GET', - 'Content-Type': 'application/xml; charset=utf-8', - 'Cache-Control': `public, max-age=${config.cache.routeExpire}`, - 'X-Content-Type-Options': 'nosniff', -}; -if (config.nodeName) { - headers['RSSHub-Node'] = config.nodeName; -} - -module.exports = async (ctx, next) => { - logger.info(`${ctx.url}, user IP: ${ctx.ips[0] || ctx.ip}`); - ctx.set(headers); - ctx.set({ - 'Access-Control-Allow-Origin': config.allowOrigin || ctx.host, - }); - - await next(); - - if (!ctx.body || typeof ctx.body !== 'string' || ctx.response.get('ETag')) { - return; - } - - const status = (ctx.status / 100) | 0; - if (2 !== status) { - return; - } - - ctx.set('ETag', etagCalculate(ctx.body.replace(/(.*)<\/lastBuildDate>/, '').replace(//, ''))); - - if (ctx.fresh) { - ctx.status = 304; - ctx.body = null; - } else { - const match = ctx.body.match(/(.*)<\/lastBuildDate>/); - if (match) { - ctx.set({ - 'Last-Modified': match[1], - }); - } - } -}; diff --git a/lib/middleware/header.test.ts b/lib/middleware/header.test.ts new file mode 100644 index 00000000000000..bda96a681f85c7 --- /dev/null +++ b/lib/middleware/header.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it, afterAll } from 'vitest'; + +process.env.NODE_NAME = 'mock'; +process.env.ALLOW_ORIGIN = 'rsshub.mock'; + +let etag; + +afterAll(() => { + delete process.env.NODE_NAME; + delete process.env.ALLOW_ORIGIN; +}); + +describe('header', () => { + it(`header`, async () => { + const app = (await import('@/app')).default; + const { config } = await import('@/config'); + const response = await app.request('/test/1'); + expect(response.headers.get('access-control-allow-origin')).toBe('rsshub.mock'); + expect(response.headers.get('access-control-allow-methods')).toBe('GET'); + expect(response.headers.get('content-type')).toBe('application/xml; charset=utf-8'); + expect(response.headers.get('cache-control')).toBe(`public, max-age=${config.cache.routeExpire}`); + expect(response.headers.get('last-modified')).toBe((await response.text()).match(/(.*)<\/lastBuildDate>/)?.[1]); + expect(response.headers.get('rsshub-node')).toBe('mock'); + expect(response.headers.get('etag')).not.toBe(undefined); + etag = response.headers.get('etag'); + }); + + it(`etag`, async () => { + const app = (await import('@/app')).default; + const response = await app.request('/test/1', { + headers: { + 'If-None-Match': etag, + }, + }); + expect(response.status).toBe(304); + expect(await response.text()).toBe(''); + expect(response.headers.get('last-modified')).toBe(null); + }); +}); diff --git a/lib/middleware/header.ts b/lib/middleware/header.ts new file mode 100644 index 00000000000000..f43d8614548f81 --- /dev/null +++ b/lib/middleware/header.ts @@ -0,0 +1,48 @@ +import { MiddlewareHandler } from 'hono'; +import etagCalculate from 'etag'; +import { config } from '@/config'; +import { Data } from '@/types'; + +const headers: Record = { + 'Access-Control-Allow-Methods': 'GET', + 'Content-Type': 'application/xml; charset=utf-8', + 'Cache-Control': `public, max-age=${config.cache.routeExpire}`, + 'X-Content-Type-Options': 'nosniff', +}; +if (config.nodeName) { + headers['RSSHub-Node'] = config.nodeName; +} + +function etagMatches(etag: string, ifNoneMatch: string | null) { + return ifNoneMatch !== null && ifNoneMatch.split(/,\s*/).includes(etag); +} + +const middleware: MiddlewareHandler = async (ctx, next) => { + for (const key in headers) { + ctx.header(key, headers[key]); + } + ctx.header('Access-Control-Allow-Origin', config.allowOrigin || new URL(ctx.req.url).host); + + await next(); + + const data: Data = ctx.get('data'); + if (!data || ctx.res.headers.get('ETag')) { + return; + } + + const lastBuildDate = data.lastBuildDate; + delete data.lastBuildDate; + const etag = etagCalculate(JSON.stringify(data)); + + ctx.header('ETag', etag); + + const ifNoneMatch = ctx.req.header('If-None-Match') ?? null; + if (etagMatches(etag, ifNoneMatch)) { + ctx.status(304); + ctx.set('no-content', true); + } else { + ctx.header('Last-Modified', lastBuildDate); + } +}; + +export default middleware; diff --git a/lib/middleware/load-on-demand.js b/lib/middleware/load-on-demand.js deleted file mode 100644 index 0c15bb42305437..00000000000000 --- a/lib/middleware/load-on-demand.js +++ /dev/null @@ -1,36 +0,0 @@ -const mount = require('koa-mount'); -const Router = require('@koa/router'); -const routes = require('../v2router'); -const loadedRoutes = new Set(); - -module.exports = function (app) { - return async function (ctx, next) { - const p = ctx.request.path.split('/').filter(Boolean); - let modName = null; - let mounted = false; - - if (p.length > 0) { - modName = p[0]; - if (!loadedRoutes.has(modName)) { - const mod = routes[modName]; - // Mount module - if (mod) { - mounted = true; - loadedRoutes.add(modName); - const router = new Router(); - mod(router); - app.use(mount(`/${modName}`, router.routes())).use(router.allowedMethods()); - } - } else { - mounted = true; - } - } - - await next(); - - // We should only add it when koa router matched - if (mounted && ctx._matchedRoute) { - ctx._matchedRoute = `/${modName}${ctx._matchedRoute}`; - } - }; -}; diff --git a/lib/middleware/logger.ts b/lib/middleware/logger.ts new file mode 100644 index 00000000000000..9d2ca59cec6b8f --- /dev/null +++ b/lib/middleware/logger.ts @@ -0,0 +1,44 @@ +import { requestMetric } from '@/utils/otel'; +import { MiddlewareHandler } from 'hono'; +import logger from '@/utils/logger'; +import { getPath, time } from '@/utils/helpers'; + +enum LogPrefix { + Outgoing = '-->', + Incoming = '<--', + Error = 'xxx', +} + +const colorStatus = (status: number) => { + const out: { [key: string]: string } = { + 7: `\u001B[35m${status}\u001B[0m`, + 5: `\u001B[31m${status}\u001B[0m`, + 4: `\u001B[33m${status}\u001B[0m`, + 3: `\u001B[36m${status}\u001B[0m`, + 2: `\u001B[32m${status}\u001B[0m`, + 1: `\u001B[32m${status}\u001B[0m`, + 0: `\u001B[33m${status}\u001B[0m`, + }; + + const calculateStatus = Math.trunc(status / 100); + + return out[calculateStatus]; +}; + +const middleware: MiddlewareHandler = async (ctx, next) => { + const { method, raw, routePath } = ctx.req; + const path = getPath(raw); + + logger.info(`${LogPrefix.Incoming} ${method} ${path}`); + + const start = Date.now(); + + await next(); + + const status = ctx.res.status; + + logger.info(`${LogPrefix.Outgoing} ${method} ${path} ${colorStatus(status)} ${time(start)}`); + requestMetric.success(Date.now() - start, { path: routePath, method, status }); +}; + +export default middleware; diff --git a/lib/middleware/onerror.js b/lib/middleware/onerror.js deleted file mode 100644 index 04c9c4c9f97836..00000000000000 --- a/lib/middleware/onerror.js +++ /dev/null @@ -1,113 +0,0 @@ -const logger = require('@/utils/logger'); -const config = require('@/config').value; -const art = require('art-template'); -const path = require('path'); - -const { RequestInProgressError } = require('@/errors'); - -let Sentry; -let gitHash; - -if (config.sentry.dsn) { - Sentry = Sentry || require('@sentry/node'); - Sentry.init({ - dsn: config.sentry.dsn, - }); - Sentry.configureScope((scope) => { - scope.setTag('node_name', config.nodeName); - }); - - logger.info('Sentry inited.'); -} - -try { - gitHash = require('git-rev-sync').short(); -} catch (e) { - gitHash = (process.env.HEROKU_SLUG_COMMIT && process.env.HEROKU_SLUG_COMMIT.slice(0, 7)) || (process.env.VERCEL_GIT_COMMIT_SHA && process.env.VERCEL_GIT_COMMIT_SHA.slice(0, 7)) || 'unknown'; -} - -module.exports = async (ctx, next) => { - try { - const time = +new Date(); - await next(); - if (config.sentry.dsn && +new Date() - time >= config.sentry.routeTimeout) { - Sentry.withScope((scope) => { - scope.setTag('route', ctx._matchedRoute); - scope.setTag('name', ctx.request.path.split('/')[1]); - scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request)); - Sentry.captureException(new Error('Route Timeout')); - }); - } - } catch (err) { - let message = err; - if (err.name && (err.name === 'HTTPError' || err.name === 'RequestError')) { - message = `${err.message}: target website might be blocking our access, you can host your own RSSHub instance for a better usability.`; - } else if (err instanceof Error) { - message = process.env.NODE_ENV === 'production' ? err.message : err.stack; - } - - logger.error(`Error in ${ctx.request.path}: ${message}`); - - if (config.isPackage) { - ctx.body = { - error: { - message: err.message ? err.message : err, - }, - }; - } else { - ctx.set({ - 'Content-Type': 'text/html; charset=UTF-8', - }); - - if (err instanceof RequestInProgressError) { - ctx.status = 503; - message = err.message; - ctx.set('Cache-Control', `public, max-age=${config.cache.requestTimeout}`); - } else if (ctx.status === 403) { - message = err.message; - } else { - ctx.status = 404; - } - - const requestPath = ctx.request.path; - - ctx.body = art(path.resolve(__dirname, '../views/error.art'), { - requestPath, - message, - errorPath: ctx.path, - nodeVersion: process.version, - gitHash, - }); - } - - if (!ctx.debug.errorPaths[ctx.request.path]) { - ctx.debug.errorPaths[ctx.request.path] = 0; - } - ctx.debug.errorPaths[ctx.request.path]++; - - if (!ctx.debug.errorRoutes[ctx._matchedRoute]) { - ctx._matchedRoute && (ctx.debug.errorRoutes[ctx._matchedRoute] = 0); - } - ctx._matchedRoute && ctx.debug.errorRoutes[ctx._matchedRoute]++; - - if (!ctx.state.debuged) { - if (!ctx.debug.routes[ctx._matchedRoute]) { - ctx._matchedRoute && (ctx.debug.routes[ctx._matchedRoute] = 0); - } - ctx._matchedRoute && ctx.debug.routes[ctx._matchedRoute]++; - - if (ctx.response.get('X-Koa-Redis-Cache') || ctx.response.get('X-Koa-Memory-Cache')) { - ctx.debug.hitCache++; - } - } - - if (config.sentry.dsn) { - Sentry.withScope((scope) => { - scope.setTag('route', ctx._matchedRoute); - scope.setTag('name', ctx.request.path.split('/')[1]); - scope.addEventProcessor((event) => Sentry.Handlers.parseRequest(event, ctx.request)); - Sentry.captureException(err); - }); - } - } -}; diff --git a/lib/middleware/parameter.js b/lib/middleware/parameter.js deleted file mode 100644 index edd0da8290ffae..00000000000000 --- a/lib/middleware/parameter.js +++ /dev/null @@ -1,327 +0,0 @@ -const entities = require('entities'); -const cheerio = require('cheerio'); -const { simplecc } = require('simplecc-wasm'); -const got = require('@/utils/got'); -const config = require('@/config').value; -const { RE2JS } = require('re2js'); - -let mercury_parser; - -const resolveRelativeLink = ($, elem, attr, baseUrl) => { - const $elem = $(elem); - - if (baseUrl) { - try { - const oldAttr = $elem.attr(attr); - if (oldAttr) { - // e.g. should leave