Merge pull request #19 from georgiana-b/feature/tender-and-network-up…

…date

Update tenders and networks

api/serializers/bid.js

@@ -16,7 +16,6 @@ const lotSerializer = require('./lot');
 
 bidSerializer.formatBid = function (bid) {
   const formattedBid = _.pick(bid, ['isWinning', 'isSubcontracted', 'xYearApproximated']);
-  formattedBid.TEDCANID = bid.xTEDCANID;
   formattedBid.value = _.get(bid, 'price.netAmountEur') || undefined;
   formattedBid.xAmountApproximated = _.get(bid, 'price.xAmountApproximated');
   return formattedBid;

api/serializers/network_actor.js

@@ -9,15 +9,15 @@ const bidSerializer = require('./bid');
 function formatNetworkActor(networkActor) {
   const formattedActor = _.pick(
     networkActor,
-    ['id', 'label', 'type', 'medianCompetition', 'value', 'country'],
+    ['id', 'label', 'type', 'medianCompetition', 'value'],
   );
   formattedActor.flags = {};
   formattedActor.hidden = !networkActor.active;
   return formattedActor;
 }
 
 function formatActorWithDetails(network, networkActor, nodeIDs) {
-  const node = _.pick(networkActor, ['label', 'id', 'type', 'medianCompetition', 'value']);
+  const node = _.pick(networkActor, ['label', 'id', 'type', 'medianCompetition', 'value', 'countries']);
   node.flags = {};
   node.hidden = !networkActor.active;
   const edgeToBidClass = networkActor.type === 'buyer' ? 'Awards' : 'Participates';

api/serializers/tender.js

@@ -18,7 +18,6 @@ tenderSerializer.formatTender = function (tender) {
   const formattedTender = _.pick(tender, ['id', 'title', 'titleEnglish', 'description', 'sources',
     'isCoveredByGpa', 'isFrameworkAgreement', 'procedureType', 'year', 'country', 'isDirective', 'xYearApproximated']);
   formattedTender.isEUFunded = tender.xIsEuFunded;
-  formattedTender.TEDCNID = tender.xTEDCNID;
   formattedTender.isDirective = tender.xIsDirective;
   formattedTender.finalValue = _.get(tender, 'finalPrice.netAmountEur') || undefined;
   formattedTender.xAmountApproximated = _.get(tender, 'finalPrice.xAmountApproximated');

api/swagger/swagger.yaml

@@ -1346,9 +1346,12 @@ definitions:
         enum:
           - bidder
           - buyer
-      country:
-        type: string
-        description: ISO 3166-1 alpha-2 country code
+      countries:
+        type: array
+        description: Array of bidder/buyer country codes
+        items:
+          type: string
+          description: ISO 3166-1 alpha-2 country code of bidder/buyer
       medianCompetition:
         type: number
         format: double
@@ -1556,9 +1559,6 @@ definitions:
       isWinning:
         type: boolean
         description: Did this bid won the lot?
-      TEDCANID:
-        type: string
-        description: ID of the Contract Award Notice on ted.europa.eu
       isSubcontracted:
         type: boolean
         description: Did the winner subcontract another company for this bid?
@@ -1629,9 +1629,6 @@ definitions:
         items:
           type: string
           description: URL to tender source
-      TEDCNID:
-        type: string
-        description: ID of the Contract Notice on ted.europa.eu
       title:
         type: string
         description: Title of the tender
@@ -1744,9 +1741,12 @@ definitions:
         enum:
           - bidder
           - buyer
-      country:
-        type: string
-        description: Actor country
+      countries:
+        type: array
+        description: Array of bidder/buyer country codes
+        items:
+          type: string
+          description: ISO 3166-1 alpha-2 country code of bidder/buyer
   Address:
     type: object
     properties:

api/writers/actor_cluster.js

@@ -19,6 +19,7 @@ async function createCluster(networkID, clusterParams) {
     label: clusterParams.label,
     type: clusterParams.type,
     active: true,
+    countries: _.uniq(calcuatedAttrs.countries),
     value: calcuatedAttrs[network.settings.nodeSize],
     medianCompetition: calcuatedAttrs.medianCompetition,
   };
@@ -114,7 +115,8 @@ function calculateCluster(edgeToBidClass, network, actorIDs) {
   const clusterQuery = `SELECT
     count(*) as numberOfWinningBids,
     sum(price.netAmountEur) as amountOfMoneyExchanged,
-    median(out('AppliedTo').bidsCount) as medianCompetition
+    median(out('AppliedTo').bidsCount) as medianCompetition,
+    unionall(in('${edgeToBidClass}').address.country) as countries
     FROM Bid
     WHERE ${_.join(networkWriters.queryToBidFilters(network.query), ' AND ')}
     AND in('${edgeToBidClass}').id in :actorIDs

api/writers/network.js

@@ -47,6 +47,83 @@ async function createNetwork(networkParams, user) {
   return transaction.commit(2).return(`$${networkName}`).one();
 }
 
+async function updateNetwork(networkParams, existingNetwork) {
+  const clusterWriters = require('./actor_cluster');
+  const newNetworkQuery = _.get(networkParams, 'query', undefined);
+  let networkQuery = {};
+
+  if (_.isUndefined(newNetworkQuery) === true) {
+    networkQuery = existingNetwork.query;
+  } else {
+    networkQuery = _.pickBy(newNetworkQuery, (val) => !(_.isUndefined(val)));
+  }
+  if (_.isEmpty(networkQuery) === true) {
+    throw new codes.BadRequestError('Network "query" can\'t be empty.');
+  }
+
+  // We have to retrieve the clusters here before we delete the actor nodes
+  const clusterQuery = `SELECT *,
+    out('Includes').in('ActingAs').id as originalActorIDs
+    FROM  ActorCluster
+    WHERE out('PartOf').id=:networkID`;
+  const existingClusters = await config.db.query(
+    clusterQuery,
+    { params: { networkID: existingNetwork.id } }
+  );
+
+  const networkName = recordName(existingNetwork.id, 'Network');
+  const newNetworkAttrs = Object.assign({}, {
+    name: _.get(networkParams, 'name', existingNetwork.name),
+    settings: _.get(networkParams, 'settings', existingNetwork.settings),
+    synopsis: _.get(networkParams, 'synopsis', existingNetwork.synopsis),
+    query: networkQuery,
+    created: existingNetwork.created,
+    updated: moment().format('YYYY-MM-DD HH:mm:ss'),
+  });
+
+  const transaction = config.db.let(networkName, (t) => {
+    t.update('Network')
+      .set(newNetworkAttrs)
+      .where({ '@rid': existingNetwork['@rid'] })
+      .return('AFTER');
+  });
+
+  await deleteNetworkActors(transaction, existingNetwork);
+
+  const networkActorsMapping = await Promise.join(
+    createBuyerNodes(transaction, newNetworkAttrs.settings, networkQuery, networkName),
+    createBidderNodes(transaction, newNetworkAttrs.settings, networkQuery, networkName),
+    (buyerActorsMapping, bidderActorsMapping) =>
+      Object.assign(buyerActorsMapping, bidderActorsMapping),
+  );
+  await Promise.all([
+    createContractsEdges(transaction, newNetworkAttrs.settings, networkQuery, networkActorsMapping),
+    createPartnersEdges(transaction, 'Awards', networkQuery, networkActorsMapping),
+    createPartnersEdges(transaction, 'Participates', networkQuery, networkActorsMapping),
+  ]);
+
+  return transaction.commit(2).return(`$${networkName}`).one()
+    .then((updatedNetwork) =>
+      Promise.map(existingClusters, (existingCluster) => {
+          const clusterActorsQuery = `SELECT * FROM NetworkActor
+            WHERE out('PartOf').id=:networkID
+            AND in('ActingAs').id in :actorIDs
+          `
+          return config.db.query(
+            clusterActorsQuery,
+            { params: { networkID: updatedNetwork.id, actorIDs: existingCluster.originalActorIDs}},
+          ).then((newClusterActors) => {
+            const newNodeIDs = _.map(newClusterActors, 'id');
+            return clusterWriters.updateCluster(
+              updatedNetwork.id,
+              existingCluster.id,
+              { nodes: newNodeIDs }
+            );
+          })
+      })
+    );
+}
+
 function queryToBidFilters(networkQuery) {
   const filters = [];
   const actorFilters = [];
@@ -91,6 +168,7 @@ function createBidderNodes(transaction, networkSettings, networkQuery, networkNa
     bidder[@rid] as bidderRID,
     count(*) as numberOfWinningBids,
     sum(price.netAmountEur) as amountOfMoneyExchanged,
+    bidder.address.country as country,
     median(out('AppliedTo').bidsCount) as medianCompetition
     FROM (
       SELECT *, in('Participates') as bidder
@@ -111,6 +189,7 @@ function createBidderNodes(transaction, networkSettings, networkQuery, networkNa
       nodeAttrs.id = uuidv4();
       nodeAttrs.type = 'bidder';
       nodeAttrs.active = true;
+      nodeAttrs.countries = [node.country];
       nodeAttrs.value = node[networkSettings.nodeSize];
       const partnerName = createNetworkActor(transaction, nodeAttrs, node.bidderRID, networkName);
       bidderActorMapping[node.bidderRID] = partnerName;
@@ -140,11 +219,12 @@ function createBuyerNodes(transaction, networkSettings, networkQuery, networkNam
     .then((buyerNodes) => Promise.map(buyerNodes, (node) => {
       const nodeAttrs = _.pick(
         node,
-        ['label', 'medianCompetition', 'country'],
+        ['label', 'medianCompetition'],
       );
       nodeAttrs.id = uuidv4();
       nodeAttrs.type = 'buyer';
       nodeAttrs.active = true;
+      nodeAttrs.countries = [node.country];
       nodeAttrs.value = node[networkSettings.nodeSize];
       const partnerName = createNetworkActor(transaction, nodeAttrs, node.buyerRID, networkName);
       buyerActorMapping[node.buyerRID] = partnerName;
@@ -259,6 +339,22 @@ function createNetworkEdge(transaction, edgeClass, edgeAttrs, fromName, toName)
   return edgeName;
 }
 
+function deleteNetworkActors(transaction, existingNetwork) {
+  return config.db.select().from('NetworkActor')
+    .where({
+      "out('PartOf').id": existingNetwork.id,
+      "@class": 'NetworkActor',
+    })
+    .all()
+    .then((networkActors) => Promise.map(networkActors, (networkActor) => {
+      const deleteActorName = recordName(networkActor.id, 'delete');
+
+      transaction.let(deleteActorName, (t) =>
+        t.delete('vertex', 'NetworkActor')
+          .where({ '@rid': networkActor['@rid'] }));
+    }));
+}
+
 module.exports = {
   createNetwork,
   createBidderNodes,
@@ -270,4 +366,5 @@ module.exports = {
   createNetworkEdge,
   queryToBidFilters,
   recordName,
+  updateNetwork,
 };

api/writers/tender.js

@@ -19,7 +19,7 @@ function recordName(id, className) {
 
 // Returns true
 // Raises OrientDBError if the writing failed
-async function writeTender(fullTenderRecord, skipMilitaryFilters = false) {
+async function writeTender(fullTenderRecord, skipMilitaryFilters = true) {
   const awardedLots = _.filter(fullTenderRecord.lots, { status: 'AWARDED' });
 
   // If there are no awarded lots don't even process the tender
@@ -60,6 +60,19 @@ async function writeTender(fullTenderRecord, skipMilitaryFilters = false) {
     }
   });
 
+
+  // The tender already exists. We're deleting the existing CPV, Buyer, Bidder edges
+  // We also delete the existing Lots and Bids before creating new ones
+  if (_.isUndefined(existingTender) === false) {
+    await deleteHasCPV(transaction, tenderName, existingTenderID)
+    await deleteCreates(transaction, tenderName, existingTenderID)
+    const existingLotRel = await config.db.select("out('Comprises')").from('Tender')
+      .where({ '@rid': existingTenderID }).one();
+    const existingLotRIDs = existingLotRel.out;
+    await Promise.map(existingLotRIDs, (existingLotRID) =>
+      deleteLot(transaction, existingLotRID));
+  }
+
   const processedCpvs = await Promise.map((fullTenderRecord.cpvs || []), (rawCpv) =>
     upsertCpv(transaction, rawCpv, existingTenderID, tenderName));
   // Only process further valid cpvs
@@ -71,14 +84,6 @@ async function writeTender(fullTenderRecord, skipMilitaryFilters = false) {
     (rawBuyer) => upsertBuyer(transaction, rawBuyer, existingTenderID, tenderName, fullTenderRecord), // eslint-disable-line max-len
   );
 
-  if (_.isUndefined(existingTender) === false) {
-    const existingLotRel = await config.db.select("out('Comprises')").from('Tender')
-      .where({ '@rid': existingTenderID }).one();
-    const existingLotRIDs = existingLotRel.out;
-    await Promise.map(existingLotRIDs, (existingLotRID) =>
-      deleteLot(transaction, existingLotRID));
-  }
-
   await Promise.map((awardedLots || []), (rawLot) => {
     const rawBids = (rawLot.bids || []);
     return createLot(transaction, rawLot, tenderName, fullTenderRecord)
@@ -94,6 +99,24 @@ async function writeTender(fullTenderRecord, skipMilitaryFilters = false) {
     });
 }
 
+async function deleteHasCPV(transaction, tenderName, existingTenderID) {
+  const operationName = `deleteHasCPV${tenderName}`;
+
+  transaction.let(operationName, (t) => 
+    t.delete('edge', 'HasCPV')
+      .where({ out: existingTenderID}));
+  return operationName;
+}
+
+async function deleteCreates(transaction, tenderName, existingTenderID){
+  const operationName = `deleteCreates${tenderName}`;
+
+  transaction.let(operationName, (t) => 
+    t.delete('edge', 'Creates')
+      .where({ in: existingTenderID}));
+  return operationName;
+}
+
 async function deleteLot(transaction, lotRID) {
   const lotName = `delete${recordName(uuidv4(), 'Lot')}`;
 
@@ -190,24 +213,11 @@ async function upsertBuyer(transaction, rawBuyer, existingTenderID, tenderName,
     });
   }
 
-  const existingRel = await config.db.select().from('Creates')
-    .where({
-      in: (existingTenderID || null),
-      out: (existingBuyerID || null),
-    }).one();
-  transaction.let(`${buyerName}creates${tenderName}`, (t) => {
-    if (_.isUndefined(existingRel)) {
-      t.create('edge', 'Creates')
-        .from(`$${buyerName}`)
-        .to(`$${tenderName}`)
-        .set(buyerExtractor.extractCreates(rawBuyer));
-    } else {
-      t.update('Creates')
-        .set(buyerExtractor.extractCreates(rawBuyer))
-        .where({ '@rid': existingRel['@rid'] })
-        .return('AFTER');
-    }
-  });
+  transaction.let(`${buyerName}creates${tenderName}`, (t) =>
+    t.create('edge', 'Creates')
+      .from(`$${buyerName}`)
+      .to(`$${tenderName}`)
+      .set(buyerExtractor.extractCreates(rawBuyer)));
   return buyerName;
 }
 
@@ -268,29 +278,13 @@ async function upsertCpv(transaction, rawCpv, existingTenderID, tenderName) {
       }
     }
 
-    const existingRel = await config.db.select().from('HasCPV')
-      .where({
-        // This is needed because undefined confuses OrientDB
-        in: (existingCpvID || null),
-        out: (existingTenderID || null),
-      }).one();
     const edgeName = `${tenderName}has${cpvName}`;
     if (_.includes(_.map(transaction._state.bcommon, (arr) => arr[0]), edgeName) === false) {
-      if (_.isUndefined(existingRel)) {
-        transaction.let(edgeName, (t) => {
-          t.create('edge', 'HasCPV')
-            .from(`$${tenderName}`)
-            .to(`$${cpvName}`)
-            .set(cpvExtractor.extractHasCpv(rawCpv));
-        });
-      } else {
-        transaction.let(edgeName, (t) => {
-          t.update('HasCPV')
-            .set(cpvExtractor.extractHasCpv(rawCpv))
-            .where({ '@rid': existingRel['@rid'] })
-            .return('AFTER');
-        });
-      }
+      transaction.let(edgeName, (t) =>
+        t.create('edge', 'HasCPV')
+          .from(`$${tenderName}`)
+          .to(`$${cpvName}`)
+          .set(cpvExtractor.extractHasCpv(rawCpv)));
     }
     return cpvName;
   }