export csv cached url and write to file when not in production

StateVoicesNational · Dec 8, 2020 · f23c989 · f23c989
1 parent 35ebb0f
commit f23c989
Show file tree

Hide file tree

Showing 6 changed files with 108 additions and 10 deletions.
diff --git a/src/api/campaign.js b/src/api/campaign.js
@@ -80,6 +80,12 @@ export const schema = gql`
     count: Int!
   }
 
+  type CampaignExportData {
+    error: String
+    campaignExportUrl: String
+    campaignMessagesExportUrl: String
+  }
+
   type Campaign {
     id: ID
     organization: Organization
@@ -109,6 +115,7 @@ export const schema = gql`
     stats: CampaignStats
     completionStats: CampaignCompletionStats
     pendingJobs: [JobRequest]
+    exportResults: CampaignExportData
     ingestMethodsAvailable: [IngestMethod]
     ingestMethod: IngestMethod
     useDynamicAssignment: Boolean

diff --git a/src/containers/AdminCampaignStats.jsx b/src/containers/AdminCampaignStats.jsx
@@ -327,6 +327,36 @@ class AdminCampaignStats extends React.Component {
             </div>
           </div>
         </div>
+        {campaign.exportResults ? (
+          <div>
+            {campaign.exportResults.error ? (
+              <div>Export failed: {campaign.exportResults.error}</div>
+            ) : null}
+            {campaign.exportResults.campaignExportUrl &&
+            campaign.exportResults.campaignExportUrl.startsWith("http") ? (
+              <div>
+                Most recent export:
+                <a href={campaign.exportResults.campaignExportUrl} download>
+                  Contacts Export CSV
+                </a>
+                <a
+                  href={campaign.exportResults.campaignMessagesExportUrl}
+                  download
+                >
+                  Messages Export CSV
+                </a>
+              </div>
+            ) : (
+              <div>
+                Local export was successful, saved on the server at:
+                <br />
+                {campaign.exportResults.campaignExportUrl}
+                <br />
+                {campaign.exportResults.campaignMessagesExportUrl}
+              </div>
+            )}
+          </div>
+        ) : null}
         {campaign.joinToken && campaign.useDynamicAssignment ? (
           <OrganizationJoinLink
             organizationUuid={campaign.joinToken}
@@ -434,6 +464,11 @@ const queries = {
             unrepliedCount: contactsCount(contactsFilter: $needsResponseFilter)
             contactsCount
           }
+          exportResults {
+            error
+            campaignExportUrl
+            campaignMessagesExportUrl
+          }
           pendingJobs {
             id
             jobType

diff --git a/src/server/api/campaign.js b/src/server/api/campaign.js
@@ -1,5 +1,5 @@
 import { accessRequired } from "./errors";
-import { mapFieldsToModel } from "./lib/utils";
+import { mapFieldsToModel, mapFieldsOrNull } from "./lib/utils";
 import { errorDescriptions } from "./lib/twilio";
 import { Campaign, JobRequest, r, cacheableData } from "../models";
 import { getUsers } from "./user";
@@ -281,6 +281,11 @@ export const resolvers = {
       return null;
     }
   },
+  CampaignExportData: mapFieldsOrNull([
+    "error",
+    "campaignExportUrl",
+    "campaignMessagesExportUrl"
+  ]),
   Campaign: {
     ...mapFieldsToModel(
       [
@@ -327,6 +332,14 @@ export const resolvers = {
     organization: async (campaign, _, { loaders }) =>
       campaign.organization ||
       loaders.organization.load(campaign.organization_id),
+    exportResults: async (campaign, _, { user }) => {
+      try {
+        await accessRequired(user, campaign.organization_id, "ADMIN", true);
+      } catch (err) {
+        return null;
+      }
+      return cacheableData.campaign.getExportData(campaign.id);
+    },
     pendingJobs: async (campaign, _, { user }) => {
       await accessRequired(
         user,

diff --git a/src/server/api/lib/utils.js b/src/server/api/lib/utils.js
@@ -1,5 +1,14 @@
 import humps from "humps";
 
+export function mapFieldsOrNull(fields) {
+  const resolvers = {};
+
+  fields.forEach(field => {
+    resolvers[field] = o => o[field] || null;
+  });
+  return resolvers;
+}
+
 export function mapFieldsToModel(fields, model) {
   const resolvers = {};
 

diff --git a/src/server/models/cacheable_queries/campaign.js b/src/server/models/cacheable_queries/campaign.js
@@ -27,6 +27,8 @@ import organizationCache from "./organization";
 const cacheKey = id => `${process.env.CACHE_PREFIX || ""}campaign-${id}`;
 const infoCacheKey = id =>
   `${process.env.CACHE_PREFIX || ""}campaigninfo-${id}`;
+const exportCampaignCacheKey = id =>
+  `${process.env.CACHE_PREFIX || ""}campaignexport-${id}`;
 
 const CONTACT_CACHE_ENABLED =
   process.env.REDIS_CONTACT_CACHE || global.REDIS_CONTACT_CACHE;
@@ -233,6 +235,26 @@ const campaignCache = {
     }
     return {};
   },
+  saveExportData: async (id, data) => {
+    if (r.redis) {
+      const exportCacheKey = exportCampaignCacheKey(id);
+      await r.redis
+        .multi()
+        .set(exportCacheKey, JSON.stringify(data))
+        .expire(exportCacheKey, 43200)
+        .execAsync();
+    }
+  },
+  getExportData: async id => {
+    if (r.redis) {
+      const exportCacheKey = exportCampaignCacheKey(id);
+      const data = await r.redis.getAsync(exportCacheKey);
+      if (data) {
+        return JSON.parse(data);
+      }
+    }
+    return null;
+  },
   updateAssignedCount: async id => {
     if (r.redis) {
       try {

diff --git a/src/workers/jobs.js b/src/workers/jobs.js
@@ -27,6 +27,7 @@ import { Notifications, sendUserNotification } from "../server/notifications";
 import { getConfig } from "../server/api/lib/config";
 import { invokeTaskFunction, Tasks } from "./tasks";
 import fs from "fs";
+import path from "path";
 
 const defensivelyDeleteOldJobsForCampaignJobType = async job => {
   console.log("job", job);
@@ -812,6 +813,7 @@ export async function exportCampaign(job) {
 
   const campaignCsv = Papa.unparse(contacts);
   const messageCsv = Papa.unparse(messages);
+  const exportResults = {};
   console.log("exportCampaign csvs", campaignCsv.length, messageCsv.length);
   if (
     getConfig("AWS_ACCESS_AVAILABLE") ||
@@ -842,6 +844,8 @@ export async function exportCampaign(job) {
         "getObject",
         params
       );
+      exportResults.campaignExportUrl = campaignExportUrl;
+      exportResults.campaignMessagesExportUrl = campaignMessagesExportUrl;
 
       await sendEmail({
         to: user.email,
@@ -857,6 +861,7 @@ export async function exportCampaign(job) {
       log.info(`Successfully exported ${id}`);
     } catch (err) {
       log.error(err);
+      exportResults.error = err.message;
       await sendEmail({
         to: user.email,
         subject: `Export failed for ${campaign.title}`,
@@ -865,18 +870,25 @@ export async function exportCampaign(job) {
       });
     }
   } else if (process.env.NODE_ENV !== "production") {
-    console.log("Writing CSV to ./");
-    fs.writeFileSync(`./campaign-export-${campaign.id}.csv`, campaignCsv);
-    fs.writeFileSync(
-      `./campaign-export-messages-${campaign.id}.csv`,
-      messageCsv
-    );
+    const contactsFile = `./campaign-export-${campaign.id}.csv`;
+    const messagesFile = `./campaign-export-${campaign.id}-message.csv`;
+    exportResults.campaignExportUrl =
+      "file://" + path.resolve("./", contactsFile);
+    exportResults.campaignMessagesExportUrl =
+      "file://" + path.resolve("./", messagesFile);
+
+    console.log(`Writing CSVs to ${contactsFile} and ${messagesFile}`);
+    fs.writeFileSync(contactsFile, campaignCsv);
+    fs.writeFileSync(messagesFile, messageCsv);
   } else {
     console.log("Would have saved the following to S3:");
-    console.log(campaignCsv);
-    console.log(messageCsv);
+    log.debug(campaignCsv);
+    log.debug(messageCsv);
+  }
+  if (exportResults.campaignExportUrl) {
+    exportResults.createdAt = String(new Date());
+    await cacheableData.campaign.saveExportData(campaign.id, exportResults);
   }
-
   await defensivelyDeleteJob(job);
 }