Feat/optional rate limiting

.env

@@ -18,8 +18,9 @@ REACT_APP_WHO_API_KEY=''
 
 # Configuration settings
 # CHROME_PATH='/usr/bin/chromium' # The path the the Chromium executable
-# PORT='3000' # Port to serve the API, when running server.js
-# DISABLE_GUI='false' # Disable the GUI, and only serve the API
-# API_TIMEOUT_LIMIT='10000' # The timeout limit for API requests, in milliseconds
-# API_CORS_ORIGIN='*' # Enable CORS, by setting your allowed hostname(s) here
-# REACT_APP_API_ENDPOINT='/api' # The endpoint for the API (can be local or remote)
+# PORT='3000'                     # Port to serve the API, when running server.js
+# DISABLE_GUI='false'             # Disable the GUI, and only serve the API
+# API_TIMEOUT_LIMIT='10000'       # The timeout limit for API requests, in milliseconds
+# API_CORS_ORIGIN='*'             # Enable CORS, by setting your allowed hostname(s) here
+# API_ENABLE_RATE_LIMIT='true'    # Enable rate limiting for the API
+# REACT_APP_API_ENDPOINT='/api'   # The endpoint for the API (can be local or remote)

.github/README.md

@@ -839,10 +839,12 @@ Key | Value
 
 Key | Value
 ---|---
-`CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`)
 `PORT` | Port to serve the API, when running server.js (e.g. `3000`)
-`DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`)
+`API_ENABLE_RATE_LIMIT` | Enable rate-limiting for the /api endpoints (e.g. `true`)
 `API_TIMEOUT_LIMIT` | The timeout limit for API requests, in milliseconds (e.g. `10000`)
+`API_CORS_ORIGIN` | Enable CORS, by setting your allowed hostname(s) here (e.g. `example.com`)
+`CHROME_PATH` | The path the Chromium executable (e.g. `/usr/bin/chromium`)
+`DISABLE_GUI` | Disable the GUI, and only serve the API (e.g. `false`)
 `REACT_APP_API_ENDPOINT` | The endpoint for the API, either local or remote (e.g. `/api`)
 
 All values are optional.

api/quality.js

@@ -5,14 +5,17 @@ const handler = async (url, event, context) => {
   const apiKey = process.env.GOOGLE_CLOUD_API_KEY;
 
   if (!apiKey) {
-    throw new Error('API key (GOOGLE_CLOUD_API_KEY) not set');
+    throw new Error(
+      'Missing Google API. You need to set the `GOOGLE_CLOUD_API_KEY` environment variable'
+    );
   }
 
-  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile&key=${apiKey}`;
+  const endpoint = `https://www.googleapis.com/pagespeedonline/v5/runPagespeed?`
+  + `url=${encodeURIComponent(url)}&category=PERFORMANCE&category=ACCESSIBILITY`
+  + `&category=BEST_PRACTICES&category=SEO&category=PWA&strategy=mobile`
+  + `&key=${apiKey}`;
 
-  const response = await axios.get(endpoint);
-
-  return response.data;
+  return (await axios.get(endpoint)).data;
 };
 
 module.exports = middleware(handler);

api/sitemap.js

@@ -39,10 +39,8 @@ const handler = async (url) => {
 
     return sitemap;
   } catch (error) {
-    // If error occurs
-    console.log(error.message);
     if (error.code === 'ECONNABORTED') {
-      return { error: 'Request timed out' };
+      return { error: 'Request timed out after 5000ms' };
     } else {
       return { error: error.message };
     }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "web-check",
-  "version": "1.0.0",
+  "version": "1.1.0",
   "private": false,
   "description": "All-in-one OSINT tool for analyzing any website",
   "repository": "github:lissy93/web-check",
@@ -45,6 +45,7 @@
     "cors": "^2.8.5",
     "csv-parser": "^3.0.0",
     "dotenv": "^16.3.1",
+    "express-rate-limit": "^7.2.0",
     "flatted": "^3.2.7",
     "follow-redirects": "^1.15.2",
     "got": "^13.0.0",

server.js

@@ -2,6 +2,7 @@ const express = require('express');
 const fs = require('fs');
 const path = require('path');
 const cors = require('cors');
+const rateLimit = require('express-rate-limit');
 const historyApiFallback = require('connect-history-api-fallback');
 require('dotenv').config();
 
@@ -20,6 +21,34 @@ app.use(cors({
   origin: process.env.API_CORS_ORIGIN || '*',
 }));
 
+// Define max requests within each time frame
+const limits = [
+  { timeFrame: 10 * 60, max: 100, messageTime: '10 minutes' },
+  { timeFrame: 60 * 60, max: 250, messageTime: '1 hour' },
+  { timeFrame: 12 * 60 * 60, max: 500, messageTime: '12 hours' },
+];
+
+// Construct a message to be returned if the user has been rate-limited
+const makeLimiterResponseMsg = (retryAfter) => {
+  const why = 'This keeps the service running smoothly for everyone. '
+  + 'You can get around these limits by running your own instance of Web Check.';
+  return `You've been rate-limited, please try again in ${retryAfter} seconds.\n${why}`;
+};
+
+// Create rate limiters for each time frame
+const limiters = limits.map(limit => rateLimit({
+  windowMs: limit.timeFrame * 1000,
+  max: limit.max,
+  standardHeaders: true,
+  legacyHeaders: false,
+  message: { error: makeLimiterResponseMsg(limit.messageTime) }
+}));
+
+// If rate-limiting enabled, then apply the limiters to the /api endpoint
+if (process.env.API_ENABLE_RATE_LIMIT === 'true') {
+  app.use('/api', limiters);
+}
+
 // Read and register each API function as an Express routes
 fs.readdirSync(dirPath, { withFileTypes: true })
   .filter(dirent => dirent.isFile() && dirent.name.endsWith('.js'))
@@ -85,7 +114,6 @@ fs.readdirSync(dirPath, { withFileTypes: true })
     await Promise.all(handlerPromises);
     res.json(results);
   });
-
 
 // Handle SPA routing
 app.use(historyApiFallback({

src/pages/Results.tsx

@@ -288,7 +288,7 @@ const Results = (): JSX.Element => {
     addressInfo: { address, addressType, expectedAddressTypes: urlTypeOnly },
     fetchRequest: () => fetch(`${api}/quality?url=${address}`)
       .then(res => parseJson(res))
-      .then(res => res?.lighthouseResult || { error: 'No Data'}),
+      .then(res => res?.lighthouseResult || { error: res.error || 'No Data' }),
   });
 
   // Get the technologies used to build site, using Wappalyzer

yarn.lock

@@ -9788,6 +9788,11 @@ [email protected]:
   dependencies:
     on-headers "^1.0.0"
 
+express-rate-limit@^7.2.0:
+  version "7.2.0"
+  resolved "https://registry.yarnpkg.com/express-rate-limit/-/express-rate-limit-7.2.0.tgz#06ce387dd5388f429cab8263c514fc07bf90a445"
+  integrity sha512-T7nul1t4TNyfZMJ7pKRKkdeVJWa2CqB8NA1P8BwYaoDI5QSBZARv5oMS43J7b7I5P+4asjVXjb7ONuwDKucahg==
+
 [email protected], express@^4.17.3:
   version "4.18.2"
   resolved "https://registry.npmjs.org/express/-/express-4.18.2.tgz"