Log objects rather than JSON strings and option for single line logs (#…

…2028)

* Log objects rather than JSON strings and option for single line logs

This reverts commit fcd914b.

* Better password stripping tests

README.md

@@ -145,6 +145,20 @@ app.listen(1337, function() {
 
 For a full list of available options, run `parse-server --help`.
 
+## Logging
+
+Parse Server will, by default, will log:
+* to the console
+* daily rotating files as new line delimited JSON
+
+Logs are also be viewable in Parse Dashboard.
+
+**Want to log each request and response?** Set the `VERBOSE` environment variable when starting `parse-server`. Usage :-  `VERBOSE='1' parse-server --appId APPLICATION_ID --masterKey MASTER_KEY`
+
+**Want logs to be in placed in other folder?** Pass the `PARSE_SERVER_LOGS_FOLDER` environment variable when starting `parse-server`. Usage :-  `PARSE_SERVER_LOGS_FOLDER='<path-to-logs-folder>' parse-server --appId APPLICATION_ID --masterKey MASTER_KEY`
+
+**Want new line delimited JSON error logs (for consumption by CloudWatch, Google Cloud Logging, etc.)?** Pass the `JSON_LOGS` environment variable when starting `parse-server`. Usage :-  `JSON_LOGS='1' parse-server --appId APPLICATION_ID --masterKey MASTER_KEY`
+
 # Documentation
 
 The full documentation for Parse Server is available in the [wiki](https://github.com/ParsePlatform/parse-server/wiki). The [Parse Server guide](https://github.com/ParsePlatform/parse-server/wiki/Parse-Server-Guide) is a good place to get started. If you're interested in developing for Parse Server, the [Development guide](https://github.com/ParsePlatform/parse-server/wiki/Development-Guide) will help you get set up.

spec/FileLoggerAdapter.spec.js

@@ -59,7 +59,10 @@ describe('verbose logs', () => {
         level: 'verbose'
       });
     }).then((results) => {
-      expect(results[1].message.includes('"password": "********"')).toEqual(true);
+      let logString = JSON.stringify(results);
+      expect(logString.match(/\*\*\*\*\*\*\*\*/g).length).not.toBe(0);
+      expect(logString.match(/moon-y/g)).toBe(null);
+
       var headers = {
         'X-Parse-Application-Id': 'test',
         'X-Parse-REST-API-Key': 'rest'
@@ -74,11 +77,16 @@ describe('verbose logs', () => {
           size: 100,
           level: 'verbose'
         }).then((results) => {
-          expect(results[1].message.includes('password=********')).toEqual(true);
+          let logString = JSON.stringify(results);
+          expect(logString.match(/\*\*\*\*\*\*\*\*/g).length).not.toBe(0);
+          expect(logString.match(/moon-y/g)).toBe(null);
           done();
         });
       });
-    });
+    }).catch((err) => {
+      fail(JSON.stringify(err));
+      done();
+    })
   });
 
   it("should not mask information in non _User class", (done) => {
@@ -92,7 +100,7 @@ describe('verbose logs', () => {
         level: 'verbose'
       });
     }).then((results) => {
-      expect(results[1].message.includes('"password": "pw"')).toEqual(true);
+      expect(results[1].body.password).toEqual("pw");
       done();
     });
   });

src/Config.js

@@ -22,6 +22,7 @@ export class Config {
     }
 
     this.applicationId = applicationId;
+    this.jsonLogs = cacheInfo.jsonLogs;
     this.masterKey = cacheInfo.masterKey;
     this.clientKey = cacheInfo.clientKey;
     this.javascriptKey = cacheInfo.javascriptKey;

src/ParseServer.js

@@ -69,6 +69,7 @@ const requiredUserFields = { fields: { ...SchemaController.defaultColumns._Defau
 //                 and delete
 // "loggerAdapter": a class like FileLoggerAdapter providing info, error,
 //                 and query
+// "jsonLogs": log as structured JSON objects
 // "databaseURI": a uri like mongodb://localhost:27017/dbname to tell us
 //          what database this Parse API connects to.
 // "cloud": relative location to cloud code to require, or a function
@@ -98,6 +99,7 @@ class ParseServer {
     filesAdapter,
     push,
     loggerAdapter,
+    jsonLogs,
     logsFolder,
     databaseURI,
     databaseOptions,
@@ -155,9 +157,7 @@ class ParseServer {
     }
 
     if (logsFolder) {
-      configureLogger({
-        logsFolder
-      })
+      configureLogger({logsFolder, jsonLogs});
     }
 
     if (cloud) {
@@ -172,7 +172,7 @@ class ParseServer {
     }
 
     if (verbose || process.env.VERBOSE || process.env.VERBOSE_PARSE_SERVER) {
-      configureLogger({level: 'silly'});
+      configureLogger({level: 'silly', jsonLogs});
     }
 
     const filesControllerAdapter = loadAdapter(filesAdapter, () => {
@@ -215,6 +215,7 @@ class ParseServer {
     })
 
     AppCache.put(appId, {
+      appId,
       masterKey: masterKey,
       serverURL: serverURL,
       collectionPrefix: collectionPrefix,
@@ -242,6 +243,7 @@ class ParseServer {
       liveQueryController: liveQueryController,
       sessionLength: Number(sessionLength),
       expireInactiveSessions: expireInactiveSessions,
+      jsonLogs,
       revokeSessionOnPasswordReset,
       databaseController,
     });
@@ -265,7 +267,7 @@ class ParseServer {
     return ParseServer.app(this.config);
   }
 
-  static app({maxUploadSize = '20mb'}) {
+  static app({maxUploadSize = '20mb', appId}) {
     // This app serves the Parse API directly.
     // It's the equivalent of https://api.parse.com/1 in the hosted Parse API.
     var api = express();
@@ -312,7 +314,7 @@ class ParseServer {
       return memo.concat(router.routes);
     }, []);
 
-    let appRouter = new PromiseRouter(routes);
+    let appRouter = new PromiseRouter(routes, appId);
 
     batch.mountOnto(appRouter);
 

src/PromiseRouter.js

@@ -5,9 +5,11 @@
 // themselves use our routing information, without disturbing express
 // components that external developers may be modifying.
 
-import express from 'express';
-import url     from 'url';
-import log     from './logger';
+import AppCache  from './cache';
+import express   from 'express';
+import url       from 'url';
+import log       from './logger';
+import {inspect} from 'util';
 
 export default class PromiseRouter {
   // Each entry should be an object with:
@@ -19,8 +21,9 @@ export default class PromiseRouter {
   //     status: optional. the http status code. defaults to 200
   //     response: a json object with the content of the response
   //     location: optional. a location header
-  constructor(routes = []) {
+  constructor(routes = [], appId) {
     this.routes = routes;
+    this.appId = appId;
     this.mountRoutes();
   }
 
@@ -107,16 +110,16 @@ export default class PromiseRouter {
     for (var route of this.routes) {
       switch(route.method) {
       case 'POST':
-        expressApp.post(route.path, makeExpressHandler(route.handler));
+        expressApp.post(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       case 'GET':
-        expressApp.get(route.path, makeExpressHandler(route.handler));
+        expressApp.get(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       case 'PUT':
-        expressApp.put(route.path, makeExpressHandler(route.handler));
+        expressApp.put(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       case 'DELETE':
-        expressApp.delete(route.path, makeExpressHandler(route.handler));
+        expressApp.delete(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       default:
         throw 'unexpected code branch';
@@ -129,16 +132,16 @@ export default class PromiseRouter {
     for (var route of this.routes) {
       switch(route.method) {
       case 'POST':
-        expressApp.post(route.path, makeExpressHandler(route.handler));
+        expressApp.post(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       case 'GET':
-        expressApp.get(route.path, makeExpressHandler(route.handler));
+        expressApp.get(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       case 'PUT':
-        expressApp.put(route.path, makeExpressHandler(route.handler));
+        expressApp.put(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       case 'DELETE':
-        expressApp.delete(route.path, makeExpressHandler(route.handler));
+        expressApp.delete(route.path, makeExpressHandler(this.appId, route.handler));
         break;
       default:
         throw 'unexpected code branch';
@@ -152,17 +155,30 @@ export default class PromiseRouter {
 // handler.
 // Express handlers should never throw; if a promise handler throws we
 // just treat it like it resolved to an error.
-function makeExpressHandler(promiseHandler) {
+function makeExpressHandler(appId, promiseHandler) {
+  let config = AppCache.get(appId);
   return function(req, res, next) {
     try {
-      log.verbose(req.method, maskSensitiveUrl(req), req.headers,
-                  JSON.stringify(maskSensitiveBody(req), null, 2));
+      let url = maskSensitiveUrl(req);
+      let body = maskSensitiveBody(req);
+      let stringifiedBody = JSON.stringify(body, null, 2);
+      log.verbose(`REQUEST for [${req.method}] ${url}: ${stringifiedBody}`, {
+        method: req.method,
+        url: url,
+        headers: req.headers,
+        body: body
+      });
       promiseHandler(req).then((result) => {
         if (!result.response && !result.location && !result.text) {
           log.error('the handler did not include a "response" or a "location" field');
           throw 'control should not get here';
         }
-        log.verbose(JSON.stringify(result, null, 2));
+
+        let stringifiedResponse = JSON.stringify(result, null, 2);
+        log.verbose(
+          `RESPONSE from [${req.method}] ${url}: ${stringifiedResponse}`,
+          {result: result}
+        );
 
         var status = result.status || 200;
         res.status(status);
@@ -186,11 +202,11 @@ function makeExpressHandler(promiseHandler) {
         }
         res.json(result.response);
       }, (e) => {
-        log.verbose('error:', e);
+        log.error(`Error generating response. ${inspect(e)}`, {error: e});
         next(e);
       });
     } catch (e) {
-      log.verbose('exception:', e);
+      log.error(`Error handling request: ${inspect(e)}`, {error: e});
       next(e);
     }
   }

src/cli/cli-definitions.js

@@ -184,6 +184,10 @@ export default {
     env: "VERBOSE",
     help: "Set the logging to verbose"
   },
+  "jsonLogs": {
+    env: "JSON_LOGS",
+    help: "Log as structured JSON objects"
+  },
   "revokeSessionOnPasswordReset": {
     env: "PARSE_SERVER_REVOKE_SESSION_ON_PASSWORD_RESET",
     help: "When a user changes their password, either through the reset password email or while logged in, all sessions are revoked if this is true. Set to false if you don't want to revoke sessions.",

src/index.js

@@ -1,4 +1,5 @@
 import ParseServer          from './ParseServer';
+import logger            from './logger';
 import S3Adapter            from 'parse-server-s3-adapter'
 import FileSystemAdapter    from 'parse-server-fs-adapter'
 import InMemoryCacheAdapter from './Adapters/Cache/InMemoryCacheAdapter'
@@ -16,4 +17,4 @@ _ParseServer.createLiveQueryServer = ParseServer.createLiveQueryServer;
 let GCSAdapter = useExternal('GCSAdapter', 'parse-server-gcs-adapter');
 
 export default ParseServer;
-export { S3Adapter, GCSAdapter, FileSystemAdapter, InMemoryCacheAdapter, TestUtils, _ParseServer as ParseServer };
+export { S3Adapter, GCSAdapter, FileSystemAdapter, InMemoryCacheAdapter, TestUtils, logger, _ParseServer as ParseServer };

src/logger.js

@@ -10,36 +10,45 @@ if (typeof process !== 'undefined' && process.env.NODE_ENV === 'test') {
 }
 
 LOGS_FOLDER = process.env.PARSE_SERVER_LOGS_FOLDER || LOGS_FOLDER;
+const JSON_LOGS = process.env.JSON_LOGS || false;
 
 let currentLogsFolder = LOGS_FOLDER;
 
-function generateTransports(level) {
+function generateTransports(level, options = {}) {
   let transports = [
-    new (DailyRotateFile)({
-      filename: 'parse-server.info',
-      dirname: currentLogsFolder,
-      name: 'parse-server',
-      level: level
-    }),
-    new (DailyRotateFile)({
-      filename: 'parse-server.err',
-      dirname: currentLogsFolder,
-      name: 'parse-server-error',
-      level: 'error'
-    })
-  ]
+    new (DailyRotateFile)(
+      Object.assign({
+        filename: 'parse-server.info',
+        dirname: currentLogsFolder,
+        name: 'parse-server',
+        level: level
+      }, options)
+    ),
+    new (DailyRotateFile)(
+      Object.assign({
+          filename: 'parse-server.err',
+          dirname: currentLogsFolder,
+          name: 'parse-server-error',
+          level: 'error'
+        }
+      ), options)
+  ];
   if (!process.env.TESTING || process.env.VERBOSE) {
-    transports = [new (winston.transports.Console)({
-      colorize: true,
-      level:level
-    })].concat(transports);
+    transports = [
+      new (winston.transports.Console)(
+        Object.assign({
+          colorize: true,
+          level: level
+        }, options)
+      )
+    ].concat(transports);
   }
   return transports;
 }
 
 const logger = new winston.Logger();
 
-export function configureLogger({logsFolder, level = winston.level}) {
+export function configureLogger({ logsFolder, jsonLogs, level = winston.level }) {
   winston.level = level;
   logsFolder = logsFolder || currentLogsFolder;
 
@@ -53,16 +62,22 @@ export function configureLogger({logsFolder, level = winston.level}) {
   }
   currentLogsFolder = logsFolder;
 
+  const options = {};
+  if (jsonLogs) {
+    options.json = true;
+    options.stringify = true;
+  }
+  const transports = generateTransports(level, options);
   logger.configure({
-    transports:  generateTransports(level)
+    transports: transports
   })
 }
 
-configureLogger({logsFolder: LOGS_FOLDER});
+configureLogger({ logsFolder: LOGS_FOLDER, jsonLogs: JSON_LOGS });
 
 export function addGroup(groupName) {
   let level = winston.level;
-  let transports =  generateTransports().concat(new (DailyRotateFile)({
+  let transports = generateTransports().concat(new (DailyRotateFile)({
     filename: groupName,
     dirname: currentLogsFolder,
     name: groupName,