From 0c5dbd66a36992ba985b82f7e1be2ac71b1ce2aa Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 14 Mar 2024 07:57:59 +0000 Subject: [PATCH 1/5] Adds express-rate-limit --- package.json | 1 + yarn.lock | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/package.json b/package.json index dcd85626..2c459db3 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "cors": "^2.8.5", "csv-parser": "^3.0.0", "dotenv": "^16.3.1", + "express-rate-limit": "^7.2.0", "flatted": "^3.2.7", "follow-redirects": "^1.15.2", "got": "^13.0.0", diff --git a/yarn.lock b/yarn.lock index 97e06771..937a2b46 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9788,6 +9788,11 @@ express-logging@1.1.1: dependencies: on-headers "^1.0.0" +express-rate-limit@^7.2.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-7.2.0.tgz#06ce387dd5388f429cab8263c514fc07bf90a445" + integrity sha512-T7nul1t4TNyfZMJ7pKRKkdeVJWa2CqB8NA1P8BwYaoDI5QSBZARv5oMS43J7b7I5P+4asjVXjb7ONuwDKucahg== + express@4.18.2, express@^4.17.3: version "4.18.2" resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz" From 41cd3798059430bbd6578bcda66bfa4cab652fa1 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 14 Mar 2024 07:58:35 +0000 Subject: [PATCH 2/5] Bumps to V 1.1.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 2c459db3..86af95ef 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "web-check", - "version": "1.0.0", + "version": "1.1.0", "private": false, "description": "All-in-one OSINT tool for analyzing any website", "repository": "github:lissy93/web-check", From ee74c5866ad9ff2c5ed02e1ab15e36b54001c377 Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 14 Mar 2024 07:59:03 +0000 Subject: [PATCH 3/5] Slight improvments to error messaging for sitemap and quality report --- api/quality.js | 13 ++++++++----- api/sitemap.js | 4 +--- src/pages/Results.tsx | 2 +- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/api/quality.js b/api/quality.js index 7f2e3f13..a49afe11 100644 --- a/api/quality.js +++ b/api/quality.js @@ -5,14 +5,17 @@ const handler = async (url, event, context) => { const apiKey = process.env.GOOGLE_CLOUD_API_KEY; if (!apiKey) { - throw new Error('API key (GOOGLE_CLOUD_API_KEY) not set'); + throw new Error( + 'Missing Google API. You need to set the `GOOGLE_CLOUD_API_KEY` environment variable' + ); } - const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`; + const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?` + + `url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY` + + `&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile` + + `&key=${apiKey}`; - const response = await axios.get(endpoint); - - return response.data; + return (await axios.get(endpoint)).data; }; module.exports = middleware(handler); diff --git a/api/sitemap.js b/api/sitemap.js index 9d77ec19..8d593e4e 100644 --- a/api/sitemap.js +++ b/api/sitemap.js @@ -39,10 +39,8 @@ const handler = async (url) => { return sitemap; } catch (error) { - // If error occurs - console.log(error.message); if (error.code === 'ECONNABORTED') { - return { error: 'Request timed out' }; + return { error: 'Request timed out after 5000ms' }; } else { return { error: error.message }; } diff --git a/src/pages/Results.tsx b/src/pages/Results.tsx index 259d91ca..31902870 100644 --- a/src/pages/Results.tsx +++ b/src/pages/Results.tsx @@ -288,7 +288,7 @@ const Results = (): JSX.Element => { addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly }, fetchRequest: () => fetch(`${api}/quality?url=${address}`) .then(res => parseJson(res)) - .then(res => res?.lighthouseResult || { error: 'No Data'}), + .then(res => res?.lighthouseResult || { error: res.error || 'No Data' }), }); // Get the technologies used to build site, using Wappalyzer From 3f80d5808509aef8ac9db8197df542bcf429cf1c Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 14 Mar 2024 07:59:19 +0000 Subject: [PATCH 4/5] Adds optional rate-limiting functionality --- server.js | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/server.js b/server.js index f0bf39d8..cc25ecf5 100644 --- a/server.js +++ b/server.js @@ -2,6 +2,7 @@ const express = require('express'); const fs = require('fs'); const path = require('path'); const cors = require('cors'); +const rateLimit = require('express-rate-limit'); const historyApiFallback = require('connect-history-api-fallback'); require('dotenv').config(); @@ -20,6 +21,34 @@ app.use(cors({ origin: process.env.API_CORS_ORIGIN || '*', })); +// Define max requests within each time frame +const limits = [ + { timeFrame: 10 * 60, max: 100, messageTime: '10 minutes' }, + { timeFrame: 60 * 60, max: 250, messageTime: '1 hour' }, + { timeFrame: 12 * 60 * 60, max: 500, messageTime: '12 hours' }, +]; + +// Construct a message to be returned if the user has been rate-limited +const makeLimiterResponseMsg = (retryAfter) => { + const why = 'This keeps the service running smoothly for everyone. ' + + 'You can get around these limits by running your own instance of Web Check.'; + return `You've been rate-limited, please try again in ${retryAfter} seconds.\n${why}`; +}; + +// Create rate limiters for each time frame +const limiters = limits.map(limit => rateLimit({ + windowMs: limit.timeFrame * 1000, + max: limit.max, + standardHeaders: true, + legacyHeaders: false, + message: { error: makeLimiterResponseMsg(limit.messageTime) } +})); + +// If rate-limiting enabled, then apply the limiters to the /api endpoint +if (process.env.API_ENABLE_RATE_LIMIT === 'true') { + app.use('/api', limiters); +} + // Read and register each API function as an Express routes fs.readdirSync(dirPath, { withFileTypes: true }) .filter(dirent => dirent.isFile() && dirent.name.endsWith('.js')) @@ -85,7 +114,6 @@ fs.readdirSync(dirPath, { withFileTypes: true }) await Promise.all(handlerPromises); res.json(results); }); - // Handle SPA routing app.use(historyApiFallback({ From f63418039adb733fe3b16dbc2b8ef82db510388d Mon Sep 17 00:00:00 2001 From: Alicia Sykes Date: Thu, 14 Mar 2024 07:59:44 +0000 Subject: [PATCH 5/5] Adds docs for rate limit env var --- .env | 11 ++++++----- .github/README.md | 6 ++++-- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.env b/.env index 8cd3a44e..7e40b561 100644 --- a/.env +++ b/.env @@ -18,8 +18,9 @@ REACT_APP_WHO_API_KEY='' # Configuration settings # CHROME_PATH='/usr/bin/chromium' # The path the the Chromium executable -# PORT='3000' # Port to serve the API, when running server.js -# DISABLE_GUI='false' # Disable the GUI, and only serve the API -# API_TIMEOUT_LIMIT='10000' # The timeout limit for API requests, in milliseconds -# API_CORS_ORIGIN='*' # Enable CORS, by setting your allowed hostname(s) here -# REACT_APP_API_ENDPOINT='/api' # The endpoint for the API (can be local or remote) +# PORT='3000' # Port to serve the API, when running server.js +# DISABLE_GUI='false' # Disable the GUI, and only serve the API +# API_TIMEOUT_LIMIT='10000' # The timeout limit for API requests, in milliseconds +# API_CORS_ORIGIN='*' # Enable CORS, by setting your allowed hostname(s) here +# API_ENABLE_RATE_LIMIT='true' # Enable rate limiting for the API +# REACT_APP_API_ENDPOINT='/api' # The endpoint for the API (can be local or remote) diff --git a/.github/README.md b/.github/README.md index f4caa0bf..d9c6f092 100644 --- a/.github/README.md +++ b/.github/README.md @@ -839,10 +839,12 @@ Key | Value Key | Value ---|--- -`CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`) `PORT` | Port to serve the API, when running server.js (e.g. `3000`) -`DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`) +`API_ENABLE_RATE_LIMIT` | Enable rate-limiting for the /api endpoints (e.g. `true`) `API_TIMEOUT_LIMIT` | The timeout limit for API requests, in milliseconds (e.g. `10000`) +`API_CORS_ORIGIN` | Enable CORS, by setting your allowed hostname(s) here (e.g. `example.com`) +`CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`) +`DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`) `REACT_APP_API_ENDPOINT` | The endpoint for the API, either local or remote (e.g. `/api`) All values are optional.