Merge pull request #6 from sofvanh/computingArgumentScores

Computing argument scores

backend/package.json

@@ -15,6 +15,7 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "@tensorflow/tfjs-node": "^4.22.0",
     "@types/express": "^4.17.21",
     "axios": "^1.7.7",
     "express": "^4.21.0",
@@ -34,4 +35,4 @@
     "dotenv": "^16.4.5",
     "ts-node-dev": "^2.0.0"
   }
-}
+}

backend/src/analysis/argumentScoreHandler.ts

@@ -0,0 +1,200 @@
+import { ReactionForGraph, getReactionsForGraph } from "../db/operations/reactionOperations";
+import { cosineSimilarityMatrix } from "../utils/math";
+
+
+interface ArgumentScore {
+  argumentId: string;
+  consensusScore: number;
+  fragmentationScore: number;
+  clarityScore: number;
+}
+
+export async function getArgumentScores(graphId: string): Promise<ArgumentScore[]> {
+  const reactionArray: ReactionForGraph[] = await getReactionsForGraph(graphId);
+  const minimumVotesUser = 3;
+  const minimumVotesArgument = 2;
+
+  // Count votes for each user and each argument
+  const userVoteCounts = new Map<string, number>();
+  const argumentVoteCounts = new Map<string, number>();
+
+  for (const reaction of reactionArray) {
+    if (reaction.type === 'agree' || reaction.type === 'disagree') {
+      userVoteCounts.set(reaction.userId, (userVoteCounts.get(reaction.userId) || 0) + 1);
+      argumentVoteCounts.set(reaction.argumentId, (argumentVoteCounts.get(reaction.argumentId) || 0) + 1);
+    }
+  }
+
+  // Filter reactions
+  const filteredReactions = reactionArray.filter(reaction => {
+    const userVoteCount = userVoteCounts.get(reaction.userId) || 0;
+    const argumentVoteCount = argumentVoteCounts.get(reaction.argumentId) || 0;
+    return userVoteCount >= minimumVotesUser && argumentVoteCount >= minimumVotesArgument;
+  });
+
+  // Create maps for user and argument indices
+  const userIndexMap = new Map<string, number>();
+  const argumentIndexMap = new Map<string, number>();
+
+  // Fill the maps
+  let userIndex = 0;
+  let argumentIndex = 0;
+
+  for (const reaction of filteredReactions) {
+    if (!userIndexMap.has(reaction.userId)) {
+      userIndexMap.set(reaction.userId, userIndex);
+      userIndex++;
+    }
+    if (!argumentIndexMap.has(reaction.argumentId)) {
+      argumentIndexMap.set(reaction.argumentId, argumentIndex);
+      argumentIndex++;
+    }
+  }
+
+  // Initialize the voting matrix and unclear matrix
+  const userCount = userIndexMap.size;
+  const argumentCount = argumentIndexMap.size;
+
+  const votingMatrix = new Array(userCount).fill(0).map(() => new Array(argumentCount).fill(0));
+  const unclearMatrix = new Array(userCount).fill(0).map(() => new Array(argumentCount).fill(0));
+
+  // Fill the matrices
+  for (const reaction of filteredReactions) {
+    const argumentIdx = argumentIndexMap.get(reaction.argumentId)!;
+    const userIdx = userIndexMap.get(reaction.userId)!;
+    if (reaction.type === 'agree') {
+      votingMatrix[userIdx][argumentIdx] = 1;
+    }
+    else if (reaction.type === 'disagree') {
+      votingMatrix[userIdx][argumentIdx] = -1;
+    }
+    else if (reaction.type === 'unclear') {
+      unclearMatrix[userIdx][argumentIdx] = 1;
+    }
+    else {
+      throw new Error('Invalid reaction type');
+    }
+  }
+
+  // Calculate the user similarity matrix
+  const userSimilarityMatrix: number[][] = cosineSimilarityMatrix(votingMatrix);
+
+  // Calculate the argument scores for each argument
+  const argumentScores: {
+    argumentId: string,
+    consensusScore: number,
+    fragmentationScore: number
+    clarityScore: number
+  }[] = [];
+
+  argumentIndexMap.forEach((argumentIndex, argumentId) => {
+    //Identify users who voted on this argument
+    const usersWhoVoted: number[] = [];
+    for (let i = 0; i < userCount; i++) {
+      if (votingMatrix[i][argumentIndex] !== 0) {
+        usersWhoVoted.push(i);
+      }
+    }
+    const votes = usersWhoVoted.map(i => votingMatrix[i][argumentIndex]);
+
+    if (usersWhoVoted.length >= 2) {
+
+      // Get smaller matrices for just the users who voted
+      const userSimilarities = usersWhoVoted.map(i =>
+        usersWhoVoted.map(j => userSimilarityMatrix[i][j])
+      );
+      const userAgreementMatrix = votes.map(voteI =>
+        votes.map(voteJ => (voteI === voteJ ? 1 : 0))
+      );
+      const userDisagreementMatrix = votes.map(voteI =>
+        votes.map(voteJ => (voteI === -voteJ ? 1 : 0))
+      );
+
+      // Compute individual user scores (to be aggregated as the final argument score later)
+      const userConsensusScores = new Array(usersWhoVoted.length).fill(0);
+      const userFragmentationScores = new Array(usersWhoVoted.length).fill(0);
+      const userUniquenessScores = new Array(usersWhoVoted.length).fill(0);
+      const userUnclearScores = new Array(usersWhoVoted.length).fill(0);
+
+      for (let i = 0; i < usersWhoVoted.length; i++) {
+        // Partition users into in-group and out-group
+        const inGroup: number[] = [];
+        const outGroup: number[] = [];
+        for (let j = 0; j < usersWhoVoted.length; j++) {
+          const similarity = userSimilarities[i][j];
+          if (similarity > 0) {
+            inGroup.push(j);
+          } else if (similarity < 0) {
+            outGroup.push(j);
+          }
+        }
+
+        // Calculate user consensus score
+        if (outGroup.length > 0) {
+          let consensusSum = 0
+          let outgroupWeightedSize = 0;
+          for (const j of outGroup) {
+            consensusSum += userAgreementMatrix[i][j] * userSimilarities[i][j];
+            outgroupWeightedSize += userSimilarities[i][j];
+          }
+          userConsensusScores[i] = consensusSum / outgroupWeightedSize;
+        }
+        else {
+          userConsensusScores[i] = 0;
+        }
+
+        // Calculate user fragmentation score
+        let fragmentationSum = 0;
+        let ingroupWeightedSize = 0;
+        for (const j of inGroup) { // inGroup.length is always > 0 (includes self)
+          fragmentationSum += userDisagreementMatrix[i][j] * userSimilarities[i][j];
+          ingroupWeightedSize += userSimilarities[i][j];
+        }
+        userFragmentationScores[i] = fragmentationSum / ingroupWeightedSize;
+
+
+        // Calculate user uniqueness score
+        userUniquenessScores[i] = 1 / ingroupWeightedSize
+
+        // Get user unclear score
+        userUnclearScores[i] = unclearMatrix[usersWhoVoted[i]][argumentIndex];
+      }
+
+      // Aggregate user scores to get argument scores
+      // Weighted average of user scores, weighted by uniqueness score
+
+      // Calculate argument consensus score
+      let weightedConsensusSum = 0;
+      let uniquenessSum = 0;
+      for (let i = 0; i < usersWhoVoted.length; i++) {
+        weightedConsensusSum += userConsensusScores[i] * userUniquenessScores[i];
+        uniquenessSum += userUniquenessScores[i];
+      }
+      const argumentConsensusScore = weightedConsensusSum / uniquenessSum;
+
+      // Calculate argument fragmentation score
+      let weightedFragmentationSum = 0;
+      for (let i = 0; i < usersWhoVoted.length; i++) {
+        weightedFragmentationSum += userFragmentationScores[i] * userUniquenessScores[i];
+      }
+      // Score is multiplied by 2 to scale it to the range [0, 1]
+      const argumentFragmentationScore = (weightedFragmentationSum / uniquenessSum) * 2;
+
+      // Calculate argument clarity score
+      let unclearSum = 0;
+      for (let i = 0; i < usersWhoVoted.length; i++) {
+        unclearSum += userUnclearScores[i] * userUniquenessScores[i];
+      }
+      const argumentClarityScore = 1 - (unclearSum / uniquenessSum);
+
+      argumentScores.push({
+        argumentId,
+        consensusScore: argumentConsensusScore,
+        fragmentationScore: argumentFragmentationScore,
+        clarityScore: argumentClarityScore
+      });
+    }
+  });
+
+  return argumentScores;
+}

backend/src/db/operations/graphOperations.ts

@@ -1,6 +1,7 @@
 import { query } from '../db';
 import { generateGraphId } from '../idGenerator';
 import { Argument, Edge, Graph } from '../../.shared/types';
+import { getArgumentScores } from '../../analysis/argumentScoreHandler';
 
 export async function createGraph(name: string, authorId: string): Promise<string> {
   const id = generateGraphId();
@@ -60,14 +61,29 @@ export async function getGraphData(graphId: string, userId: string): Promise<Gra
     });
   }
 
+  // Get argument scores
+  // TODO: Make sure this is efficient
+  const argumentScores = await getArgumentScores(graphId);
+  const scoresMap = new Map(
+    argumentScores.map(score => [
+      score.argumentId,
+      {
+        consensus: score.consensusScore,
+        fragmentation: score.fragmentationScore,
+        clarity: score.clarityScore
+      }
+    ])
+  );
+
   const args: Argument[] = argumentsResult.rows.map((row: { id: string; graph_id: string; statement: string; embedding: number[], author_id: string }) => ({
     id: row.id,
     graphId: row.graph_id,
     statement: row.statement,
     embedding: row.embedding,
     authorId: row.author_id,
     reactionCounts: reactionCountsMap.get(row.id) || { agree: 0, disagree: 0, unclear: 0 },
-    userReaction: userReactionsMap.get(row.id)
+    userReaction: userReactionsMap.get(row.id),
+    score: scoresMap.get(row.id)
   }));
 
   const links: Edge[] = edgesResult.rows.map((row: { id: string; graph_id: string; source_id: string; target_id: string }) => ({

backend/src/db/operations/reactionOperations.ts

@@ -37,4 +37,28 @@ export async function removeReaction(
     'DELETE FROM reactions WHERE user_id = $1 AND argument_id = $2 AND type = $3',
     [userId, argumentId, type]
   );
+}
+
+export interface ReactionForGraph {
+  userId: string;
+  argumentId: string;
+  type: string;
+}
+
+export async function getReactionsForGraph(
+  graphId: string
+): Promise<ReactionForGraph[]> {
+  const result = await query(
+    `SELECT user_id, argument_id, type
+     FROM reactions
+     JOIN arguments ON reactions.argument_id = arguments.id
+     WHERE arguments.graph_id = $1`,
+    [graphId]
+  );
+
+  return result.rows.map((row: any) => ({
+    userId: row.user_id,
+    argumentId: row.argument_id,
+    type: row.type
+  }));
 }

backend/src/embeddingHandler.ts

@@ -1,6 +1,7 @@
 import axios from 'axios';
 import { Edge, Graph } from './.shared/types';
 import { generateEdgeId } from './db/idGenerator';
+import { cosineSimilarity } from './utils/math';
 import config from './config';
 
 export function generateTopKSimilarEdges(graph: Graph, k = 2): Edge[] {
@@ -45,18 +46,6 @@ export function generateTopKSimilarEdges(graph: Graph, k = 2): Edge[] {
   }));
 }
 
-function cosineSimilarity(embedding1: number[], embedding2: number[]) {
-  if (embedding1.length !== embedding2.length) {
-    throw new Error('Embeddings must have the same length');
-  }
-
-  const dotProduct = embedding1.reduce((acc, cur, i) => acc + cur * embedding2[i], 0);
-  const magnitude1 = Math.sqrt(embedding1.reduce((acc, cur) => acc + cur * cur, 0));
-  const magnitude2 = Math.sqrt(embedding2.reduce((acc, cur) => acc + cur * cur, 0));
-
-  return dotProduct / (magnitude1 * magnitude2);
-}
-
 export async function embedText(texts: string[]) {
   const openAI_api_key = config.openAIKey;
   if (!openAI_api_key) {

backend/src/utils/math.ts

@@ -0,0 +1,43 @@
+import * as tf from '@tensorflow/tfjs-node';
+
+export function cosineSimilarity(vector1: number[], vector2: number[]) {
+    if (vector1.length !== vector2.length) {
+      throw new Error('Embeddings must have the same length');
+    }
+
+    // Convert arrays to tensors
+    const tensor1 = tf.tensor1d(vector1);
+    const tensor2 = tf.tensor1d(vector2);
+
+    // Calculate dot product and magnitudes using TensorFlow
+    const dotProduct = tf.dot(tensor1, tensor2).dataSync()[0];
+    const magnitude1 = tf.norm(tensor1).dataSync()[0];
+    const magnitude2 = tf.norm(tensor2).dataSync()[0];
+
+    // Dispose tensors to free memory
+    tensor1.dispose();
+    tensor2.dispose();
+
+    return dotProduct / (magnitude1 * magnitude2);
+  }
+
+export function cosineSimilarityMatrix(matrix: number[][]) {
+    if (matrix.length === 0 || matrix[0].length === 0) {
+      return [[]];
+    }
+    const matrixTensor = tf.tensor2d(matrix);
+    const dotProduct = tf.matMul(matrixTensor, matrixTensor.transpose());
+
+    const magnitude = tf.sqrt(tf.sum(tf.square(matrixTensor), 1, true));
+    const magnitudeProduct = tf.matMul(magnitude, magnitude.transpose());
+
+    const similarity = dotProduct.div(magnitudeProduct);
+
+    // Dispose tensors to free memory
+    matrixTensor.dispose();
+    dotProduct.dispose();
+    magnitude.dispose();
+    magnitudeProduct.dispose();
+
+    return similarity.arraySync() as number[][];
+  }