[WIP] API Key flow

apiadmin.js

@@ -0,0 +1,49 @@
+const async = require('async');
+const redis = require('./store/redis');
+const db = require('./store/db');
+
+function storeUsageCounts(cursor) {
+
+  redis.scan(cursor, "MATCH", "api_count_limit:*", (err, results) => {
+    if (err) {
+      return console.log("[ERROR] ", err);
+    }
+
+    let cursor = results[0];
+
+    async.parallel({
+      usage: (cb) => async.mapLimit(results[1], 20, (e, cb2) => redis.get(e, cb2), cb),
+      keyInfo: (cb) => async.mapLimit(results[1], 20, (e, cb2) => {
+        db.from('api_keys').where({
+          api_key: e.replace('api_count_limit:', "")
+        }).asCallback(cb2);
+      }, cb)
+    },
+    (err, results) => {
+      if( err) {
+        return console.error("[ERROR] ", err);
+      }
+
+      db('api_key_usage')
+        .insert(results.keyInfo.map((e, i) => {
+          return {
+              account_id: e[0].account_id,
+              api_key: e[0].api_key,
+              customer_id: e[0].customer_id,
+              usage_count: results.usage[i]
+          };
+        }))
+        .asCallback((err, results) => {
+          if (err) {
+            return console.error("[ERROR] ", err);
+          }
+
+          if (cursor !== "0") {
+            storeUsageCounts(cursor);
+          }
+        });
+    });
+  });
+}
+
+setInterval(() => storeUsageCounts(0), 10 * 60 * 1000); //Every 10 minutes

config.js

@@ -26,6 +26,9 @@ const defaults = {
   SESSION_SECRET: 'secret to encrypt cookies with', // string to encrypt cookies
   COOKIE_DOMAIN: '',
   GOAL: 5, // The cheese goal
+  API_PRICE: 1,
+  API_UNIT: 1,
+  API_FREE_LIMIT: 25000,
   STRIPE_SECRET: '', // for donations, in web
   STRIPE_PUBLIC: '',
   BRAIN_TREE_MERCHANT_ID: '',

index.js

@@ -10,6 +10,7 @@ const config = require('./config');
 const redis = require('./store/redis');
 const db = require('./store/db');
 const donate = require('./routes/donate');
+const api = require('./routes/api');
 const session = require('cookie-session');
 const moment = require('moment');
 const async = require('async');
@@ -71,29 +72,14 @@ app.use((req, res, cb) => {
 });
 
 app.use('/', donate(db, redis));
+app.use('/api', api(db, redis));
 app.use((req, res, next) => {
   const err = new Error('Not Found');
   err.status = 404;
   return next(err);
 });
-app.use((err, req, res, next) => {
-  console.error(err);
-  res.status(err.status || 500);
-  redis.zadd('error_500', moment().format('X'), req.originalUrl);
-  if (req.originalUrl.indexOf('/api') === 0) {
-    return res.json({
-      error: err,
-    });
-  } else if (config.NODE_ENV === 'development') {
-    // default express handler
-    next(err);
-  } else {
-    return res.render(`error/${err.status === 404 ? '404' : '500'}`, {
-      error: err,
-    });
-  }
-});
+
 const port = config.CARRY_PORT;
 const server = app.listen(port, () => {
-  console.log('[WEB] listening on %s', port);
+  console.log('[CARRY] listening on %s', port);
 });

invoice.js

@@ -0,0 +1,160 @@
+const config = require('./config');
+const db = require('./store/db');
+const moment = require('moment');
+const async = require('async');
+
+const stripe_secret = config.STRIPE_SECRET;
+const stripe_public = config.STRIPE_PUBLIC;
+const API_PRICE = config.API_PRICE;
+const API_UNIT = config.API_UNIT;
+const API_FREE_LIMIT = config.API_FREE_LIMIT;
+const stripe = require('stripe')(stripe_secret);
+
+let invoiceMonth = moment(); //.subtract(1, 'month');
+
+console.log("[METADATA] Running invoice script on", moment().format("YYYY MM DD"));
+console.log("[METADATA] Invoice is for", invoiceMonth.format("MMMM YYYY"));
+
+let countProcessed = 0, countSkipped = 0, countFailed = 0, countCharged = 0;
+
+db.raw(
+  `
+    SELECT
+      SUM(usage_count) as usage_count,
+      ARRAY_AGG(api_key) as api_jeys,
+      ARRAY_AGG(customer_id) as customer_ids,
+      account_id
+    FROM (  
+      SELECT
+        MAX(usage_count) as usage_count,
+        account_id,
+        customer_id,
+        api_key
+      FROM api_key_usage
+      WHERE
+        timestamp <= '${invoiceMonth.endOf('month').format("YYYY-MM-DD")}'
+        AND timestamp >= '${invoiceMonth.startOf('month').format("YYYY-MM-DD")}'
+      GROUP BY 2, 3, 4
+    ) as T1
+    GROUP BY 4
+  `)
+.asCallback((err, results) => {
+  if (err) {
+    return console.error(err);
+  }
+
+  console.log(results.rows);
+  process.exit(1);
+
+  async.eachLimit(results.rows, 10, (e, cb) => {
+
+    db.raw(
+    `
+      SELECT
+        MAX(timestamp),
+        usage_count,
+        account_id,
+        customer_id,
+        api_key
+      FROM api_key_usage
+      WHERE
+        timestamp <= '${invoiceMonth.endOf('month').format("YYYY-MM-DD")}'
+        AND timestamp >= '${invoiceMonth.startOf('month').format("YYYY-MM-DD")}'
+        AND account_id = ${e.account_id ? "'" + e.account_id + "'" : null}
+        AND customer_id = ${e.customer_id ? "'" + e.customer_id + "'" : null}
+        AND api_key = ${e.api_key ? "'" + e.api_key + "'" : null}
+      GROUP BY 2, 3, 4, 5
+    `)
+    .asCallback((err, results) => {
+
+      console.log("[PROCESSING] Key:", e.api_key, "| Usage:", e.usage_count, "| Account:", e.account_id, "| Customer:", e.customer_id);
+
+      countProcessed++;
+
+      if (err) {
+        console.error(err);
+        return cb(err);
+      }
+
+      if (results.rows.length === 1 && results.rows[0].usage_count === e.usage_count) {
+
+        e.usage_count = 25001;
+        if (e.usage_count <= API_FREE_LIMIT) {
+          console.log("[SKIPPED] Key", e.api_key, "under limit.");
+          countSkipped++;
+          return cb();
+        }
+
+        let chargeCount = e.usage_count - API_FREE_LIMIT;
+        let charge = Math.round(chargeCount / API_UNIT * API_PRICE * 100);
+
+        if (charge < 50) {
+          console.log("[SKIPPED] Key", e.api_key, "charge less than $0.50.");
+          countSkipped++;
+          return cb();
+        }
+
+        stripe.charges.create({
+          amount: charge,
+          currency: "usd",
+          customer: e.customer_id,
+          description: `OpenDota API usage for ${invoiceMonth.format("YYYY-MM")}. # Calls: ${chargeCount}.`,
+          metadata: {
+            api_key: e.api_key,
+            account_id: e.account_id,
+            usage: e.usage_count,
+            charge_count: chargeCount
+          }
+        }, (err, charge) => {
+          if (err) {
+            console.error("[FAILED] Charge creation failed. api_key",
+              e.api_key,
+              "account_id",
+              e.account_id,
+              "customer_id",
+              e.customer_id
+            );
+
+            console.error(err);
+            return cb(err);
+          }
+
+          console.log("[CHARGED]", charge, "| ID:", charge.id, "Key:", e.api_key, "| Usage:", e.usage_count, "| Account:", e.account_id, "| Customer:", e.customer_id);
+
+          countCharged++;
+          return cb();
+        });
+      } else {
+        if (results.rows.length != 1) {
+          console.error("[FAILED] Got multiple records. api_key",
+            e.api_key,
+            "account_id",
+            e.account_id,
+            "customer_id",
+            e.customer_id
+          );
+        } else {
+          console.error(
+            "[FAILED] Usage did not match count ad end of month. api_key",
+            e.api_key,
+            "account_id",
+            e.account_id,
+            "customer_id",
+            e.customer_id
+          );
+        }
+
+        countFailed++;
+        return cb();
+      }
+    })
+  },
+  (err) => {
+    if (err) {
+      process.exit(1);
+    }
+
+    console.log("[METADATA] Processed:", countProcessed, "| Charged:", countCharged, "| Skipped:", countSkipped, "| Failed:", countFailed);
+    process.exit(0);
+  });
+})

manifest.json

@@ -0,0 +1,20 @@
+{
+  "apps": [
+    {
+      "script": "index.js",
+      "watch": true,
+      "ignore_watch": [".git", "node_modules"],
+      "group": "backend",
+      "exec_mode": "cluster",
+      "instances": 1
+    },
+    {
+      "script": "apiadmin.js",
+      "watch": true,
+      "ignore_watch": [".git", "node_modules"],
+      "group": "backend",
+      "exec_mode": "cluster",
+      "instances": 1
+    }
+  ]
+}