Fix reporting issues (#748)

* Fix reporting issues * Fix #744 `last successful fetch was null` in log * Fix #745 No Signal message if fetching fails with HTTP response code 302 Found * Fix #746 `fetch` URL path is marked in green * Update CSP documentation Signed-off-by: Marvin A. Ruder <[email protected]> * Add PostgreSQL client and configure `psql` command Signed-off-by: Marvin A. Ruder <[email protected]> --------- Signed-off-by: Marvin A. Ruder <[email protected]>
marvinruder · Nov 23, 2023 · 4264506 · 4264506
1 parent 7ead54b
commit 4264506
Show file tree

Hide file tree

Showing 6 changed files with 51 additions and 19 deletions.
diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile
@@ -19,6 +19,7 @@ RUN \
   gpgsm \
   htop \
   openssh \
+  postgresql-client \
   zsh \
   zsh-autosuggestions \
   zsh-syntax-highlighting \
@@ -46,5 +47,7 @@ RUN \
   echo '[[ "$TERM_PROGRAM" == "vscode" ]] && . "$(code --locate-shell-integration-path zsh)"' >> /root/.zshrc && \
   echo '[[ ! -f ~/.p10k.zsh ]] || source ~/.p10k.zsh' >> /root/.zshrc && \
   find /usr/share/zsh/plugins/zsh-*/zsh-*.zsh | grep -v .plugin.zsh | sed 's/^/source /' >> /root/.zshrc && \
+  echo 'alias rtpsql="psql -h postgres -U rating-tracker"' >> /root/.zshrc && \
+  echo -e "\o /dev/null\nSET search_path TO \"rating-tracker\";\n\o" > /root/.psqlrc && \
   wget -O /tmp/rust-analyzer-no-server.vsix https://github.com/rust-lang/rust-analyzer/releases/latest/download/rust-analyzer-no-server.vsix && \
   wget -O - https://raw.githubusercontent.com/nektos/act/master/install.sh | bash
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -24,6 +24,10 @@ services:
       POSTGRES_USER: rating-tracker
       POSTGRES_PASS: rating-tracker
       LOG_LEVEL: trace
+      # for `psql` / `rtpsql` commands:
+      PGHOST: postgres
+      PGUSER: rating-tracker
+      PGPASSWORD: rating-tracker
     volumes:
       - ..:/rating-tracker
       - ./motd:/etc/motd

diff --git a/README.md b/README.md
@@ -258,7 +258,7 @@ After setting up NGINX as a webserver with SSL, the following virtual host confi
 <summary>View NGINX configuration</summary>
 
 ```nginx
-add_header "Content-Security-Policy" "default-src 'self' 'wasm-unsafe-eval'; worker-src 'self' blob:; img-src 'self' data:; style-src-elem 'self' 'unsafe-inline' rsms.me; style-src rsms.me; font-src rsms.me; frame-ancestors 'none'; form-action 'self'; base-uri 'none';";add_header "Strict-Transport-Security" "max-age=31536000; includeSubDomains" always;
+add_header "Content-Security-Policy" "default-src 'self' 'wasm-unsafe-eval'; worker-src 'self' blob:; img-src 'self' data:; style-src-elem 'self' 'unsafe-inline' rsms.me; style-src 'self' 'unsafe-inline' rsms.me; font-src rsms.me; frame-ancestors 'none'; form-action 'self'; base-uri 'none';";add_header "Strict-Transport-Security" "max-age=31536000; includeSubDomains" always;
 add_header "X-Frame-Options" "DENY";
 add_header "X-Content-Type-Options" "nosniff";
 add_header "Referrer-Policy" "same-origin";

diff --git a/packages/backend/src/fetchers/fetchHelper.ts b/packages/backend/src/fetchers/fetchHelper.ts
@@ -1,5 +1,5 @@
 import {
-  DataProvider,
+  IndividualDataProvider,
   Stock,
   dataProviderID,
   dataProviderLastFetch,
@@ -56,9 +56,9 @@ export type HTMLFetcher = (
 //   driver: WebDriver,
 // ) => Promise<boolean>;
 
-const htmlDataProviders: readonly DataProvider[] = ["morningstar", "marketScreener", "msci", "sp"] as const;
-const jsonDataProviders: readonly DataProvider[] = ["refinitiv"] as const;
-// const seleniumDataProviders: readonly DataProvider[] = [] as const;
+const htmlDataProviders: readonly IndividualDataProvider[] = ["morningstar", "marketScreener", "msci", "sp"] as const;
+const jsonDataProviders: readonly IndividualDataProvider[] = ["refinitiv"] as const;
+// const seleniumDataProviders: readonly IndividualDataProvider[] = [] as const;
 
 /**
  * An object holding the source of a fetcher. Only one of the properties is set.
@@ -83,20 +83,22 @@ type FetcherSource = {
  * be a {@link Document} or a {@link Object}.
  *
  * @param {Stock} stock the affected stock
- * @param {DataProvider} dataProvider the name of the data provider
+ * @param {IndividualDataProvider} dataProvider the name of the data provider
  * @param {FetcherSource} source the source of the fetcher
  * @returns {Promise<string>} A string holding a general informational message and a URL to the screenshot
  */
 export const captureFetchError = async (
   stock: Stock,
-  dataProvider: DataProvider,
+  dataProvider: IndividualDataProvider,
   source: FetcherSource,
 ): Promise<string> => {
   let resourceID: string = "";
   switch (true) {
     case jsonDataProviders.includes(dataProvider):
       resourceID = `error-${dataProvider}-${stock.ticker}-${new Date().getTime().toString()}.json`;
       if (
+        // Only create the resource if we actually have a JSON object
+        !source?.json ||
         // Create the JSON resource in Redis
         !(await createResource(
           {
@@ -112,6 +114,8 @@ export const captureFetchError = async (
     case htmlDataProviders.includes(dataProvider):
       resourceID = `error-${dataProvider}-${stock.ticker}-${new Date().getTime().toString()}.html`;
       if (
+        // Only create the resource if we actually have a valid HTML document
+        !source?.document?.documentElement?.toString() ||
         // Create the HTML resource in Redis
         !(await createResource(
           {
@@ -139,7 +143,7 @@ export const captureFetchError = async (
 /**
  * A record of functions that extract data from a data provider.
  */
-const dataProviderFetchers: Partial<Record<DataProvider, Fetcher>> = {
+const dataProviderFetchers: Record<IndividualDataProvider, Fetcher> = {
   morningstar: morningstarFetcher,
   marketScreener: marketScreenerFetcher,
   msci: msciFetcher,
@@ -192,10 +196,14 @@ const determineConcurrency = (req: Request): number => {
  *
  * @param {Request} req Request object
  * @param {Response} res Response object
- * @param {DataProvider} dataProvider The data provider to fetch from
+ * @param {IndividualDataProvider} dataProvider The data provider to fetch from
  * @throws an {@link APIError} in case of a severe error
  */
-export const fetchFromDataProvider = async (req: Request, res: Response, dataProvider: DataProvider): Promise<void> => {
+export const fetchFromDataProvider = async (
+  req: Request,
+  res: Response,
+  dataProvider: IndividualDataProvider,
+): Promise<void> => {
   let stockList: Stock[];
 
   if (req.query.ticker) {
@@ -275,7 +283,7 @@ export const fetchFromDataProvider = async (req: Request, res: Response, dataPro
               `Stock ${stock.ticker}: Skipping ${
                 dataProviderName[dataProvider]
               } fetch since last successful fetch was ${DateTime.fromJSDate(
-                stock[dataProviderLastFetch[dataProvider]].getTime(),
+                stock[dataProviderLastFetch[dataProvider]],
               ).toRelative()}`,
             );
             stocks.skipped.push(stock);
@@ -380,14 +388,14 @@ export const fetchFromDataProvider = async (req: Request, res: Response, dataPro
  * @param {string} url The URL to fetch from
  * @param {AxiosRequestConfig} config The Axios request configuration
  * @param {Stock} stock The affected stock
- * @param {DataProvider} dataProvider The name of the data provider to fetch from
+ * @param {IndividualDataProvider} dataProvider The name of the data provider to fetch from
  * @returns {Document} The parsed HTML document
  */
 export const getAndParseHTML = async (
   url: string,
   config: AxiosRequestConfig,
   stock: Stock,
-  dataProvider: DataProvider,
+  dataProvider: IndividualDataProvider,
 ): Promise<Document> =>
   new DOMParser({
     errorHandler: {

diff --git a/packages/backend/src/utils/logFormatterConfig.ts b/packages/backend/src/utils/logFormatterConfig.ts
@@ -320,10 +320,14 @@ export const pinoPrettyConfig: PrettyOptions = process.env.PLAIN_LOG
                   { ...getPrefixFromRawPrefix({ ip: req.ip }), textColor: "magentaBright" },
                 ],
                 [
-                  // HTTP method and URL path
-                  req.method,
-                  ...req.url.slice(1, req.url.indexOf("?") == -1 ? undefined : req.url.indexOf("?")).split("/"),
-                ].map((rawPrefix) => ({ icon: rawPrefix, color: prefixColors[rawPrefix] ?? "grey" })),
+                  // HTTP method
+                  { icon: req.method, color: prefixColors[req.method] ?? "grey" },
+                  // URL path
+                  ...req.url
+                    .slice(1, req.url.indexOf("?") == -1 ? undefined : req.url.indexOf("?"))
+                    .split("/")
+                    .map((rawPrefix) => ({ icon: rawPrefix, color: "grey" })),
+                ],
                 // Cookies
                 Object.entries(req.cookies).length
                   ? prettifyRecord(req.cookies, { icon: prefixIcons.cookie, color: "grey", textColor: "yellow" })

diff --git a/packages/commons/src/lib/DataProvider.ts b/packages/commons/src/lib/DataProvider.ts
@@ -25,6 +25,16 @@ export const dataProviderArray = [
  */
 export type DataProvider = (typeof dataProviderArray)[number];
 
+/**
+ * A data provider from which stocks are fetched individually.
+ */
+export type IndividualDataProvider = Exclude<DataProvider, "sustainalytics">;
+
+/**
+ * A data provider from which stocks are fetched in bulk.
+ */
+export type BulkDataProvider = "sustainalytics";
+
 /**
  * Checks if a string is a valid data provider.
  *
@@ -60,9 +70,12 @@ export const dataProviderID: Record<DataProvider, keyof Stock> = {
 };
 
 /**
- * The stock last fetch dates related to the data providers.
+ * The stock last fetch date properties related to the data providers.
  */
-export const dataProviderLastFetch: Record<Exclude<DataProvider, "sustainalytics">, keyof Stock> = {
+export const dataProviderLastFetch: Record<
+  IndividualDataProvider,
+  "morningstarLastFetch" | "marketScreenerLastFetch" | "msciLastFetch" | "refinitivLastFetch" | "spLastFetch"
+> = {
   morningstar: "morningstarLastFetch",
   marketScreener: "marketScreenerLastFetch",
   msci: "msciLastFetch",