Reduce value set of Analyst Consensus (#1291)

* Add `analystRatings` stock property * Add Analyst Rating Bar * Simplify error handling in fetchers * Add helpers for mathematical operations on Records * Remove Selenium-related code * Run more tests concurrently Signed-off-by: Marvin A. Ruder <[email protected]>
marvinruder · May 14, 2024 · 16c8ecc · 16c8ecc
1 parent 760779c
commit 16c8ecc
Show file tree

Hide file tree

Showing 75 changed files with 990 additions and 1,044 deletions.
diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml
@@ -14,7 +14,6 @@ services:
     environment:
       DATABASE_URL: postgresql://rating-tracker:rating-tracker@postgres:5432/rating-tracker?schema=rating-tracker
       REDIS_URL: redis://redis:6379
-      # SELENIUM_URL: http://selenium:4444
       SIGNAL_URL: http://signal:8080
       PORT: 3001
       MAX_FETCH_CONCURRENCY: 4

diff --git a/packages/backend/dev/docker-compose.yml b/packages/backend/dev/docker-compose.yml
@@ -59,21 +59,6 @@ services:
       - ./postgresql/pgpass:/var/lib/postgresql/pgpass
     shm_size: "256mb"
 
-  # selenium:
-  #   container_name: selenium
-  #   hostname: selenium
-  #   image: seleniarm/standalone-chromium
-  #   restart: unless-stopped
-  #   environment:
-  #     - SE_NODE_MAX_SESSIONS=2
-  #   cap_drop:
-  #     - all
-  #   security_opt:
-  #     - no-new-privileges
-  #   ports:
-  #     - "127.0.0.1:4444:4444"
-  #   shm_size: "2gb"
-
   signal:
     container_name: signal
     hostname: signal

diff --git a/packages/backend/package.json b/packages/backend/package.json
@@ -46,12 +46,14 @@
     "pino": "9.1.0",
     "pino-pretty": "11.0.0",
     "prisma": "5.14.0",
+    "prisma-json-types-generator": "3.0.4",
     "redis": "4.6.13",
     "redis-om": "0.4.3",
     "response-time": "2.3.2",
     "supertest": "7.0.0",
     "swagger-ui-dist": "5.17.9",
     "swagger-ui-express": "5.0.0",
+    "typescript": "5.4.5",
     "undici": "6.16.1",
     "vite": "5.2.11",
     "vitest": "1.6.0",
@@ -75,7 +77,6 @@
     "eslint-plugin-import": "2.29.1",
     "eslint-plugin-jsdoc": "48.2.4",
     "eslint-plugin-prettier": "5.1.3",
-    "prettier": "3.2.5",
-    "typescript": "5.4.5"
+    "prettier": "3.2.5"
   }
 }
diff --git a/...ges/backend/prisma/migrations/05-1234-reduce-value-set-of-analyst-consensus/migration.sql b/...ges/backend/prisma/migrations/05-1234-reduce-value-set-of-analyst-consensus/migration.sql
@@ -0,0 +1,7 @@
+-- CreateEnum
+CREATE TYPE "AnalystRating" AS ENUM ('Sell', 'Underperform', 'Hold', 'Outperform', 'Buy');
+
+-- AlterTable
+ALTER TABLE "Stock" ADD COLUMN     "analystRatings" JSONB,
+DROP COLUMN "analystConsensus",
+ADD COLUMN     "analystConsensus" "AnalystRating";
diff --git a/packages/backend/prisma/schema.prisma b/packages/backend/prisma/schema.prisma
@@ -1,10 +1,15 @@
 generator client {
-  provider      = "prisma-client-js"
-  output        = "./client"
-  binaryTargets = ["native", "linux-musl-openssl-3.0.x", "linux-musl-arm64-openssl-3.0.x"]
+  provider        = "prisma-client-js"
+  output          = "./client"
+  binaryTargets   = ["native", "linux-musl-openssl-3.0.x", "linux-musl-arm64-openssl-3.0.x"]
   previewFeatures = ["omitApi"]
 }
 
+generator json {
+  provider  = "prisma-json-types-generator"
+  namespace = "PrismaJSON"
+}
+
 datasource db {
   provider = "postgresql"
   url      = env("DATABASE_URL")
@@ -35,7 +40,9 @@ model Stock {
   positionIn52w                             Float?
   marketScreenerID                          String?              @unique @db.VarChar(255)
   marketScreenerLastFetch                   DateTime?            @db.Timestamp(6)
-  analystConsensus                          Float?
+  analystConsensus                          AnalystRating?
+  /// [AnalystRatings]
+  analystRatings                            Json?
   analystCount                              Int?                 @db.SmallInt
   analystTargetPrice                        Float?
   analystTargetPricePercentageToLastClose   Float?
@@ -689,3 +696,11 @@ enum MSCIESGRating {
   B
   CCC
 }
+
+enum AnalystRating {
+  Sell
+  Underperform
+  Hold
+  Outperform
+  Buy
+}
diff --git a/packages/backend/src/controllers/FetchController.ts b/packages/backend/src/controllers/FetchController.ts
@@ -392,7 +392,7 @@ class FetchController extends Singleton {
         successfulCount += 1;
       } catch (e) {
         if (req.query.ticker)
-          // If this request was for a single stock, we shut down the driver and throw an error.
+          // If this request was for a single stock, we throw an error.
           throw new APIError(
             (e as APIError).status ?? 500,
             `Stock ${stock.ticker}: Unable to extract Sustainalytics ESG Risk`,

diff --git a/packages/backend/src/controllers/StatusController.live.test.ts b/packages/backend/src/controllers/StatusController.live.test.ts
@@ -19,7 +19,6 @@ tests.push({
     expect(res.body.services).not.toHaveProperty("Redis");
 
     // Mocked to be not ready during live tests:
-    // expect(res.body.services).toHaveProperty("Selenium");
     expect(res.body.services).toHaveProperty("Signal");
     expect(res.body.services.Signal).toBe("Signal is not ready");
   },

diff --git a/packages/backend/src/controllers/StatusController.ts b/packages/backend/src/controllers/StatusController.ts
@@ -9,7 +9,6 @@ import { redisIsReady } from "../redis/redis";
 import { signalIsReadyOrUnused } from "../signal/signalBase";
 import Endpoint from "../utils/Endpoint";
 import Singleton from "../utils/Singleton";
-// import { seleniumIsReady } from "../utils/webdriver";
 
 /**
  * This class is responsible for providing a status report of the backend API and the services it depends on.
@@ -40,7 +39,6 @@ class StatusController extends Singleton {
         // The order is important here and must match the order in `serviceArray`.
         prismaIsReady(),
         redisIsReady(),
-        // seleniumIsReady(),
         signalIsReadyOrUnused(),
       ])
     ).forEach((result, index) => {

diff --git a/packages/backend/src/controllers/StocksController.live.test.ts b/packages/backend/src/controllers/StocksController.live.test.ts
@@ -10,6 +10,7 @@ import {
   styleArray,
   watchlistsEndpointPath,
   portfoliosEndpointPath,
+  analystRatingArray,
 } from "@rating-tracker/commons";
 import type { Response } from "supertest";
 
@@ -185,11 +186,11 @@ tests.push({
     expectStocksToBePresent(
       await supertest
         .get(
-          `${baseURL}${stocksEndpointPath}?analystConsensusMin=7&analystConsensusMax=8.5` +
-            "&analystCountMin=20&analystCountMax=40&sortBy=name",
+          `${baseURL}${stocksEndpointPath}?analystConsensusMin=Outperform&analystConsensusMax=Outperform` +
+            "&analystCountMin=25&analystCountMax=40&sortBy=name",
         )
         .set("Cookie", ["authToken=exampleSessionID"]),
-      ["Iberdrola", "MercadoLibre", "Novo Nordisk", "Ørsted A/S"],
+      ["Novo Nordisk"],
     );
   },
 });
@@ -285,12 +286,13 @@ tests.push({
       .get(`${baseURL}${stocksEndpointPath}?totalScoreMin=30&totalScoreMax=60&sortBy=name`)
       .set("Cookie", ["authToken=exampleSessionID"]);
     expect(res.status).toBe(200);
-    expect(res.body.count).toBe(4);
-    expect(res.body.stocks).toHaveLength(4);
+    expect(res.body.count).toBe(5);
+    expect(res.body.stocks).toHaveLength(5);
     expect(res.body.stocks[0].name).toMatch("Allianz");
     expect(res.body.stocks[1].name).toMatch("Danone");
     expect(res.body.stocks[2].name).toMatch("Kion");
-    expect(res.body.stocks[3].name).toMatch("Ørsted");
+    expect(res.body.stocks[3].name).toMatch("Newmont");
+    expect(res.body.stocks[4].name).toMatch("Ørsted");
   },
 });
 
@@ -394,6 +396,9 @@ tests.push({
             case "style":
               sortCriterionArray = styleArray;
               break;
+            case "analystConsensus":
+              sortCriterionArray = analystRatingArray;
+              break;
             case "msciESGRating":
               sortCriterionArray = msciESGRatingArray;
               break;

diff --git a/packages/backend/src/controllers/StocksController.ts b/packages/backend/src/controllers/StocksController.ts
@@ -14,6 +14,8 @@ import {
   stockLogoEndpointSuffix,
   WRITE_STOCKS_ACCESS,
   DUMMY_SVG,
+  isAnalystRating,
+  analystRatingArray,
 } from "@rating-tracker/commons";
 import type { Request, RequestHandler, Response } from "express";
 
@@ -250,14 +252,25 @@ class StocksController extends Singleton {
         filters.push({ morningstarFairValuePercentageToLastClose: { lte: morningstarFairValueDiffMax } });
     }
 
-    if (req.query.analystConsensusMin !== undefined) {
-      const analystConsensusMin = Number(req.query.analystConsensusMin);
-      if (!Number.isNaN(analystConsensusMin)) filters.push({ analystConsensus: { gte: analystConsensusMin } });
+    let analystConsensusArray = [...analystRatingArray];
+    if (req.query.analystConsensusMin !== undefined && typeof req.query.analystConsensusMin === "string") {
+      const analystConsensusMin = req.query.analystConsensusMin;
+      if (isAnalystRating(analystConsensusMin))
+        analystConsensusArray = analystConsensusArray.filter(
+          (analystRating) =>
+            analystRatingArray.indexOf(analystRating) >= analystRatingArray.indexOf(analystConsensusMin),
+        );
     }
-    if (req.query.analystConsensusMax !== undefined) {
-      const analystConsensusMax = Number(req.query.analystConsensusMax);
-      if (!Number.isNaN(analystConsensusMax)) filters.push({ analystConsensus: { lte: analystConsensusMax } });
+    if (req.query.analystConsensusMax !== undefined && typeof req.query.analystConsensusMax === "string") {
+      const analystConsensusMax = req.query.analystConsensusMax;
+      if (isAnalystRating(analystConsensusMax))
+        analystConsensusArray = analystConsensusArray.filter(
+          (analystRating) =>
+            analystRatingArray.indexOf(analystRating) <= analystRatingArray.indexOf(analystConsensusMax),
+        );
     }
+    if (analystRatingArray.some((msciESGRating) => !analystConsensusArray.includes(msciESGRating)))
+      filters.push({ analystConsensus: { in: analystConsensusArray } });
 
     if (req.query.analystCountMin !== undefined) {
       const analystCountMin = Number(req.query.analystCountMin);

diff --git a/packages/backend/src/db/tables/stockTable.live.test.ts b/packages/backend/src/db/tables/stockTable.live.test.ts
@@ -41,7 +41,7 @@ tests.push({
       high52w: 145.67,
       marketScreenerID: "NEW-STOCK-238712974",
       marketScreenerLastFetch: new Date(),
-      analystConsensus: 2.5,
+      analystConsensus: "Hold",
       analystCount: 5,
       analystTargetPrice: 150,
       msciID: "IID000001238712974",
@@ -66,7 +66,7 @@ tests.push({
     const slightlyWorseValues: Partial<Omit<Stock, "ticker">> = {
       starRating: 3,
       morningstarFairValue: 150,
-      analystConsensus: 2.3,
+      analystConsensus: "Underperform",
       analystTargetPrice: 145,
       msciESGRating: "CCC",
       msciTemperature: 2.2,

diff --git a/packages/backend/src/db/tables/stockTable.ts b/packages/backend/src/db/tables/stockTable.ts
@@ -1,5 +1,5 @@
-import type { Stock, MSCIESGRating, OmitDynamicAttributesStock } from "@rating-tracker/commons";
-import { msciESGRatingArray } from "@rating-tracker/commons";
+import type { Stock, MSCIESGRating, OmitDynamicAttributesStock, AnalystRating } from "@rating-tracker/commons";
+import { analystRatingArray, msciESGRatingArray } from "@rating-tracker/commons";
 
 import type { Prisma } from "../../../prisma/client";
 import { addDynamicAttributesToStockData, dynamicStockAttributes } from "../../models/dynamicStockAttributes";
@@ -145,6 +145,17 @@ export const updateStock = async (ticker: string, newValues: Partial<Stock>, for
         case "sustainalyticsESGRisk":
           let signalPrefix = "";
           switch (k) {
+            case "analystConsensus":
+              signalPrefix =
+                // larger index in array [Sell, ..., Buy] is better
+                (newValues.analystConsensus ? analystRatingArray.indexOf(newValues.analystConsensus) : -1) >
+                (stock.analystConsensus
+                  ? analystRatingArray.indexOf(stock.analystConsensus as AnalystRating)
+                  : /* c8 ignore next */ // This never occurs with our test dataset
+                    -1)
+                  ? SIGNAL_PREFIX_BETTER
+                  : SIGNAL_PREFIX_WORSE;
+              break;
             case "msciESGRating":
               signalPrefix =
                 // smaller index in array [AAA, ..., CCC] is better

diff --git a/packages/backend/src/db/types.d.ts b/packages/backend/src/db/types.d.ts
@@ -0,0 +1,7 @@
+import type { AnalystRating } from "@rating-tracker/commons";
+
+declare global {
+  namespace PrismaJSON {
+    type AnalystRatings = Record<AnalystRating, number>;
+  }
+}