From e5c5812256f30591da45681c01142a9d5d7e9323 Mon Sep 17 00:00:00 2001 From: Mathieu ANCELIN Date: Tue, 26 Nov 2019 14:55:48 +0100 Subject: [PATCH] Fix #336, fix #335, fix #334, fix #337 --- otoroshi/app/actions/api.scala | 1 + otoroshi/app/actions/backoffice.scala | 4 +- otoroshi/app/actions/privateapps.scala | 2 + .../app/controllers/AnalyticsController.scala | 3 + otoroshi/app/controllers/ApiController.scala | 139 +++++++++---- .../app/controllers/Auth0Controller.scala | 23 +-- .../controllers/BackOfficeController.scala | 16 +- otoroshi/app/controllers/U2FController.scala | 21 +- otoroshi/app/el/el.scala | 53 +++-- otoroshi/app/events/alerts.scala | 194 +++++++++++++++++- otoroshi/app/events/analytics.scala | 53 ++++- otoroshi/app/events/audit.scala | 24 ++- .../app/events/impl/ElasticAnalytics.scala | 3 +- .../app/events/impl/WebHookAnalytics.scala | 12 +- otoroshi/app/gateway/handlers.scala | 153 +------------- otoroshi/app/gateway/websockets.scala | 12 +- otoroshi/app/models/JWTVerifier.scala | 7 +- otoroshi/app/models/config.scala | 65 +++++- otoroshi/app/models/descriptor.scala | 4 +- otoroshi/app/plugins/body.scala | 22 +- .../inmemory/InMemoryAlertDataStore.scala | 6 +- .../inmemory/InMemoryAuditDataStore.scala | 6 +- .../InMemoryHealthCheckDataStore.scala | 8 +- .../storage/redis/RedisAlertDataStore.scala | 6 +- .../storage/redis/RedisAuditDataStore.scala | 6 +- .../redis/RedisHealthCheckDataStore.scala | 9 +- otoroshi/app/utils/headers.scala | 55 ++--- otoroshi/test/functional/AnalyticsSpec.scala | 2 +- 28 files changed, 592 insertions(+), 317 deletions(-) diff --git a/otoroshi/app/actions/api.scala b/otoroshi/app/actions/api.scala index 3acf0a53e7..6f555d31ae 100644 --- a/otoroshi/app/actions/api.scala +++ b/otoroshi/app/actions/api.scala @@ -21,6 +21,7 @@ case class ApiActionContext[A](apiKey: ApiKey, request: Request[A]) { .get(env.Headers.OtoroshiAdminProfile) .flatMap(p => Try(Json.parse(new String(Base64.getDecoder.decode(p), Charsets.UTF_8))).toOption) def from: String = request.headers.get("X-Forwarded-For").getOrElse(request.remoteAddress) + def ua: String = request.headers.get("User-Agent").getOrElse("none") } class ApiAction(val parser: BodyParser[AnyContent])(implicit env: Env) diff --git a/otoroshi/app/actions/backoffice.scala b/otoroshi/app/actions/backoffice.scala index 9178621710..08f4112968 100644 --- a/otoroshi/app/actions/backoffice.scala +++ b/otoroshi/app/actions/backoffice.scala @@ -21,10 +21,12 @@ import scala.concurrent.{ExecutionContext, Future} case class BackOfficeActionContext[A](request: Request[A], user: Option[BackOfficeUser]) { def connected: Boolean = user.isDefined def from: String = request.headers.get("X-Forwarded-For").getOrElse(request.remoteAddress) + def ua: String = request.headers.get("User-Agent").getOrElse("none") } case class BackOfficeActionContextAuth[A](request: Request[A], user: BackOfficeUser) { def from: String = request.headers.get("X-Forwarded-For").getOrElse(request.remoteAddress) + def ua: String = request.headers.get("User-Agent").getOrElse("none") } class BackOfficeAction(val parser: BodyParser[AnyContent])(implicit env: Env) @@ -82,7 +84,7 @@ class BackOfficeActionAuth(val parser: BodyParser[AnyContent])(implicit env: Env case Some(user) => { env.datastores.backOfficeUserDataStore.blacklisted(user.email).flatMap { case true => { - Alerts.send(BlackListedBackOfficeUserAlert(env.snowflakeGenerator.nextIdStr(), env.env, user)) + Alerts.send(BlackListedBackOfficeUserAlert(env.snowflakeGenerator.nextIdStr(), env.env, user, request.headers.get("X-Forwarded-For").getOrElse(request.remoteAddress), request.headers.get("User-Agent").getOrElse("none"))) FastFuture.successful( Results.NotFound(views.html.otoroshi.error("Error", env)).removingFromSession("bousr")(request) ) diff --git a/otoroshi/app/actions/privateapps.scala b/otoroshi/app/actions/privateapps.scala index e557a1a8bb..a6776138b1 100644 --- a/otoroshi/app/actions/privateapps.scala +++ b/otoroshi/app/actions/privateapps.scala @@ -13,6 +13,8 @@ case class PrivateAppsActionContext[A](request: Request[A], user: Option[PrivateAppsUser], globalConfig: models.GlobalConfig) { def connected: Boolean = user.isDefined + def from: String = request.headers.get("X-Forwarded-For").getOrElse(request.remoteAddress) + def ua: String = request.headers.get("User-Agent").getOrElse("none") } class PrivateAppsAction(val parser: BodyParser[AnyContent])(implicit env: Env) diff --git a/otoroshi/app/controllers/AnalyticsController.scala b/otoroshi/app/controllers/AnalyticsController.scala index bff1f78649..5c80d2c138 100644 --- a/otoroshi/app/controllers/AnalyticsController.scala +++ b/otoroshi/app/controllers/AnalyticsController.scala @@ -50,6 +50,7 @@ class AnalyticsController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction "ACCESS_SERVICE_STATS", s"User accessed a service descriptor stats", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) @@ -150,6 +151,7 @@ class AnalyticsController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction "ACCESS_GLOBAL_STATS", s"User accessed a global stats", ctx.from, + ctx.ua, Json.obj() ) ) @@ -245,6 +247,7 @@ class AnalyticsController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction "ACCESS_SERVICE_EVENTS", s"User accessed a service descriptor events", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) diff --git a/otoroshi/app/controllers/ApiController.scala b/otoroshi/app/controllers/ApiController.scala index 4e4124b4fd..4c7d6b8802 100644 --- a/otoroshi/app/controllers/ApiController.scala +++ b/otoroshi/app/controllers/ApiController.scala @@ -151,7 +151,8 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ctx.user, "ACCESS_GLOBAL_LIVESTATS", "User accessed global livestats", - ctx.from + ctx.from, + ctx.ua ) ) for { @@ -192,7 +193,8 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ctx.user, "ACCESS_HOST_METRICS", "User accessed global livestats", - ctx.from + ctx.from, + ctx.ua ) ) val appEnv = Option(System.getenv("APP_ENV")).getOrElse("--") @@ -248,6 +250,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_LIVESTATS", "User accessed service livestats", ctx.from, + ctx.ua, Json.obj("serviceId" -> id) ) ) @@ -329,7 +332,8 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ctx.user, "ACCESS_ALL_LINES", "User accessed all lines", - ctx.from + ctx.from, + ctx.ua ) ) env.datastores.globalConfigDataStore.allEnv().map { @@ -351,6 +355,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICES_FOR_LINES", s"User accessed service list for line $line", ctx.from, + ctx.ua, Json.obj("line" -> line) ) ) @@ -371,7 +376,8 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ctx.user, "ACCESS_GLOBAL_CONFIG", s"User accessed global Otoroshi config", - ctx.from + ctx.from, + ctx.ua ) ) Ok(ak.toJson) @@ -393,11 +399,12 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_GLOBAL_CONFIG", s"User updated global Otoroshi config", ctx.from, + ctx.ua, ctx.request.body ) Audit.send(admEvt) Alerts.send( - GlobalConfigModification(env.snowflakeGenerator.nextIdStr(), env.env, user, conf.toJson, ak.toJson, admEvt) + GlobalConfigModification(env.snowflakeGenerator.nextIdStr(), env.env, user, conf.toJson, ak.toJson, admEvt, ctx.from, ctx.ua) ) ak.save().map(_ => Ok(Json.obj("done" -> true))) // TODO : rework } @@ -422,11 +429,12 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_GLOBAL_CONFIG", s"User updated global Otoroshi config", ctx.from, + ctx.ua, ctx.request.body ) Audit.send(admEvt) Alerts.send( - GlobalConfigModification(env.snowflakeGenerator.nextIdStr(), env.env, user, conf.toJson, ak.toJson, admEvt) + GlobalConfigModification(env.snowflakeGenerator.nextIdStr(), env.env, user, conf.toJson, ak.toJson, admEvt, ctx.from, ctx.ua) ) ak.save().map(_ => Ok(Json.obj("done" -> true))) // TODO : rework } @@ -454,6 +462,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "CREATE_SERVICE_GROUP", s"User created a service group", ctx.from, + ctx.ua, body ) Audit.send(event) @@ -461,7 +470,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceGroupCreatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) Ok(group.toJson) } @@ -487,6 +496,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_SERVICE_GROUP", s"User updated a service group", ctx.from, + ctx.ua, ctx.request.body ) Audit.send(event) @@ -494,7 +504,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceGroupUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newGroup.save().map(_ => Ok(newGroup.toJson)) } @@ -523,6 +533,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_SERVICE_GROUP", s"User updated a service group", ctx.from, + ctx.ua, ctx.request.body ) Audit.send(event) @@ -530,7 +541,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceGroupUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newGroup.save().map(_ => Ok(newGroup.toJson)) } @@ -552,6 +563,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_SERVICE_GROUP", s"User deleted a service group", ctx.from, + ctx.ua, Json.obj("serviceGroupId" -> serviceGroupId) ) Audit.send(event) @@ -559,7 +571,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceGroupDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) Ok(Json.obj("deleted" -> res)) } @@ -594,7 +606,8 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ctx.user, "ACCESS_ALL_SERVICES_GROUPS", s"User accessed all services groups", - ctx.from + ctx.from, + ctx.ua ) ) val id: Option[String] = ctx.request.queryString.get("id").flatMap(_.headOption) @@ -633,6 +646,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICES_GROUP", s"User accessed a service group", ctx.from, + ctx.ua, Json.obj("serviceGroupId" -> serviceGroupId) ) ) @@ -658,6 +672,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICES_FROM_SERVICES_GROUP", s"User accessed all services from a services group", ctx.from, + ctx.ua, Json.obj("serviceGroupId" -> serviceGroupId) ) ) @@ -723,6 +738,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "CREATE_SERVICE", s"User created a service", ctx.from, + ctx.ua, desc.toJson ) Audit.send(event) @@ -730,7 +746,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceCreatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) ServiceDescriptorQuery(desc.subdomain, desc.env, desc.domain, desc.root).addServices(Seq(desc)) Ok(desc.toJson) @@ -757,6 +773,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_SERVICE", s"User updated a service", ctx.from, + ctx.ua, desc.toJson ) Audit.send(event) @@ -764,7 +781,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) if (desc.canary.enabled && !newDesc.canary.enabled) { env.datastores.canaryDataStore.destroyCanarySession(newDesc.id) @@ -808,6 +825,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_SERVICE", s"User updated a service", ctx.from, + ctx.ua, desc.toJson ) Audit.send(event) @@ -815,7 +833,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) if (desc.canary.enabled && !newDesc.canary.enabled) { env.datastores.canaryDataStore.destroyCanarySession(newDesc.id) @@ -851,6 +869,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_SERVICE", s"User deleted a service", ctx.from, + ctx.ua, desc.toJson ) Audit.send(admEvt) @@ -858,7 +877,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ServiceDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - admEvt) + admEvt, ctx.from, ctx.ua) ) env.datastores.canaryDataStore.destroyCanarySession(desc.id) ServiceDescriptorQuery(desc.subdomain, desc.env, desc.domain, desc.root).remServices(Seq(desc)) @@ -898,6 +917,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_ALL_SERVICES", s"User accessed all service descriptors", ctx.from, + ctx.ua, Json.obj( "env" -> JsString(_env.getOrElse("--")), "group" -> JsString(_env.getOrElse("--")) @@ -944,6 +964,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE", s"User accessed a service descriptor", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) @@ -965,6 +986,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_TARGETS", s"User accessed a service targets", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) @@ -986,6 +1008,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_SERVICE_TARGETS", s"User updated a service targets", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "patch" -> body) ) val actualTargets = JsArray(desc.targets.map(t => JsString(s"${t.scheme}://${t.host}"))) @@ -999,7 +1022,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: val newDesc = desc.copy(targets = newTargets) Audit.send(event) Alerts.send( - ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event) + ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event, ctx.from, ctx.ua) ) ServiceDescriptorQuery(desc.subdomain, desc.env, desc.domain, desc.root).remServices(Seq(desc)) newDesc.save().map { _ => @@ -1024,6 +1047,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_SERVICE_TARGETS", s"User updated a service targets", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "patch" -> body) ) val newTargets = (body \ "target").asOpt[String] match { @@ -1039,7 +1063,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: val newDesc = desc.copy(targets = newTargets) Audit.send(event) Alerts.send( - ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event) + ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event, ctx.from, ctx.ua) ) ServiceDescriptorQuery(desc.subdomain, desc.env, desc.domain, desc.root).remServices(Seq(desc)) newDesc.save().map { _ => @@ -1064,6 +1088,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_SERVICE_TARGET", s"User deleted a service target", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "patch" -> body) ) val newTargets = (body \ "target").asOpt[String] match { @@ -1079,7 +1104,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: val newDesc = desc.copy(targets = newTargets) Audit.send(event) Alerts.send( - ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event) + ServiceUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event, ctx.from, ctx.ua) ) ServiceDescriptorQuery(desc.subdomain, desc.env, desc.domain, desc.root).remServices(Seq(desc)) newDesc.save().map { _ => @@ -1101,6 +1126,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_LIVESTATS", s"User accessed a service descriptor livestats", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) @@ -1145,12 +1171,13 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_HEALTH", s"User accessed a service descriptor helth", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) env.datastores.healthCheckDataStore .findAll(desc) - .map(evts => Ok(JsArray(evts.drop(paginationPosition).take(paginationPageSize).map(_.toEnrichedJson)))) + .map(evts => Ok(JsArray(evts.drop(paginationPosition).take(paginationPageSize).map(_.toJson)))) // .map(_.toEnrichedJson)))) } } } @@ -1189,6 +1216,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_ERROR_TEMPLATE", s"User updated an error template", ctx.from, + ctx.ua, errorTemplate.toJson ) Audit.send(event) @@ -1222,6 +1250,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "CREATE_ERROR_TEMPLATE", s"User created an error template", ctx.from, + ctx.ua, errorTemplate.toJson ) Audit.send(event) @@ -1249,6 +1278,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_ERROR_TEMPLATE", s"User deleted an error template", ctx.from, + ctx.ua, errorTemplate.toJson ) Audit.send(event) @@ -1291,6 +1321,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "CREATE_APIKEY", s"User created an ApiKey", ctx.from, + ctx.ua, Json.obj( "desc" -> desc.toJson, "apikey" -> apiKey.toJson @@ -1301,7 +1332,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyCreatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) env.datastores.apiKeyDataStore.addFastLookupByService(serviceId, apiKey).map { _ => env.datastores.apiKeyDataStore.findAll() @@ -1342,6 +1373,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "CREATE_APIKEY", s"User created an ApiKey", ctx.from, + ctx.ua, Json.obj( "group" -> group.toJson, "apikey" -> apiKey.toJson @@ -1352,7 +1384,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyCreatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) env.datastores.apiKeyDataStore.addFastLookupByGroup(groupId, apiKey).map { _ => env.datastores.apiKeyDataStore.findAll() @@ -1389,6 +1421,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_APIKEY", s"User updated an ApiKey", ctx.from, + ctx.ua, Json.obj( "desc" -> desc.toJson, "apikey" -> apiKey.toJson @@ -1399,7 +1432,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newApiKey.save().map(_ => Ok(newApiKey.toJson)) } @@ -1436,6 +1469,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_APIKEY", s"User updated an ApiKey", ctx.from, + ctx.ua, Json.obj( "desc" -> desc.toJson, "apikey" -> apiKey.toJson @@ -1446,7 +1480,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newApiKey.save().map(_ => Ok(newApiKey.toJson)) } @@ -1478,6 +1512,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_APIKEY", s"User updated an ApiKey", ctx.from, + ctx.ua, Json.obj( "group" -> group.toJson, "apikey" -> apiKey.toJson @@ -1488,7 +1523,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newApiKey.save().map(_ => Ok(newApiKey.toJson)) } @@ -1523,6 +1558,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_APIKEY", s"User updated an ApiKey", ctx.from, + ctx.ua, Json.obj( "group" -> group.toJson, "apikey" -> apiKey.toJson @@ -1533,7 +1569,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newApiKey.save().map(_ => Ok(newApiKey.toJson)) } @@ -1560,6 +1596,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_APIKEY", s"User deleted an ApiKey", ctx.from, + ctx.ua, Json.obj( "group" -> group.toJson, "apikey" -> apiKey.toJson @@ -1570,7 +1607,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) env.datastores.apiKeyDataStore.deleteFastLookupByGroup(groupId, apiKey) apiKey.delete().map(res => Ok(Json.obj("deleted" -> true))) @@ -1598,6 +1635,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_APIKEY", s"User deleted an ApiKey", ctx.from, + ctx.ua, Json.obj( "desc" -> desc.toJson, "apikey" -> apiKey.toJson @@ -1608,7 +1646,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) env.datastores.apiKeyDataStore.deleteFastLookupByService(serviceId, apiKey) apiKey.delete().map(res => Ok(Json.obj("deleted" -> true))) @@ -1639,6 +1677,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEYS", s"User accessed apikeys from a service descriptor", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId) ) ) @@ -1687,6 +1726,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEYS", s"User accessed apikeys from a group", ctx.from, + ctx.ua, Json.obj("groupId" -> groupId) ) ) @@ -1722,7 +1762,8 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ctx.user, "ACCESS_ALL_APIKEYS", s"User accessed all apikeys", - ctx.from + ctx.from, + ctx.ua ) ) val paginationPage: Int = ctx.request.queryString.get("page").flatMap(_.headOption).map(_.toInt).getOrElse(1) @@ -1775,6 +1816,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEY", s"User accessed an apikey from a service descriptor", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "clientId" -> clientId) ) ) @@ -1802,6 +1844,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEY", s"User accessed an apikey from a service descriptor", ctx.from, + ctx.ua, Json.obj("groupId" -> groupId, "clientId" -> clientId) ) ) @@ -1834,6 +1877,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEY_GROUP", s"User accessed an apikey servicegroup from a service descriptor", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "clientId" -> clientId) ) ) @@ -1868,6 +1912,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_APIKEY", s"User updated an ApiKey", ctx.from, + ctx.ua, Json.obj( "desc" -> desc.toJson, "apikey" -> apiKey.toJson @@ -1878,7 +1923,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: ApiKeyUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newApiKey.save().map(_ => Ok(newApiKey.toJson)) } @@ -1907,6 +1952,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEY_QUOTAS", s"User accessed an apikey quotas from a service descriptor", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "clientId" -> clientId) ) ) @@ -1936,6 +1982,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "RESET_SERVICE_APIKEY_QUOTAS", s"User reset an apikey quotas for a service descriptor", ctx.from, + ctx.ua, Json.obj("serviceId" -> serviceId, "clientId" -> clientId) ) ) @@ -1963,6 +2010,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_SERVICE_APIKEY_QUOTAS", s"User accessed an apikey quotas from a service descriptor", ctx.from, + ctx.ua, Json.obj("groupId" -> groupId, "clientId" -> clientId) ) ) @@ -1990,6 +2038,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "RESET_SERVICE_APIKEY_QUOTAS", s"User accessed an apikey quotas from a service descriptor", ctx.from, + ctx.ua, Json.obj("groupId" -> groupId, "clientId" -> clientId) ) ) @@ -2030,11 +2079,12 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "FULL_OTOROSHI_EXPORT", s"Admin exported Otoroshi", ctx.from, + ctx.ua, e ) Audit.send(event) Alerts.send( - OtoroshiExportAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(Json.obj()), event, e) + OtoroshiExportAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(Json.obj()), event, e, ctx.from, ctx.ua) ) Ok(Json.prettyPrint(e)).as("application/json") } @@ -2074,6 +2124,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "STARTED_SNOWMONKEY", s"User started snowmonkey", ctx.from, + ctx.ua, Json.obj() ) Audit.send(event) @@ -2081,7 +2132,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: SnowMonkeyStartedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) Ok(Json.obj("done" -> true)) } @@ -2097,6 +2148,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "STOPPED_SNOWMONKEY", s"User stopped snowmonkey", ctx.from, + ctx.ua, Json.obj() ) Audit.send(event) @@ -2104,7 +2156,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: SnowMonkeyStoppedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) Ok(Json.obj("done" -> true)) } @@ -2135,6 +2187,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATED_SNOWMONKEY_CONFIG", s"User updated snowmonkey config", ctx.from, + ctx.ua, config.asJson ) Audit.send(event) @@ -2142,7 +2195,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: SnowMonkeyConfigUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) Ok(config.asJson) } @@ -2166,6 +2219,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "PATCH_SNOWMONKEY_CONFIG", s"User patched snowmonkey config", ctx.from, + ctx.ua, newSnowMonkeyConfigJson ) Audit.send(event) @@ -2173,7 +2227,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: SnowMonkeyConfigUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newSnowMonkeyConfig.save().map(_ => Ok(newSnowMonkeyConfig.asJson)) } @@ -2191,11 +2245,12 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "RESET_SNOWMONKEY_OUTAGES", s"User reset snowmonkey outages for the day", ctx.from, + ctx.ua, Json.obj() ) Audit.send(event) Alerts.send( - SnowMonkeyResetAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event) + SnowMonkeyResetAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event, ctx.from, ctx.ua) ) Ok(Json.obj("done" -> true)) } @@ -2389,6 +2444,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "CREATE_CERTIFICATE", s"User created a certificate", ctx.from, + ctx.ua, body ) Audit.send(event) @@ -2396,7 +2452,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: CertCreatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) Ok(group.toJson) } @@ -2422,6 +2478,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_CERTIFICATE", s"User updated a certificate", ctx.from, + ctx.ua, ctx.request.body ) Audit.send(event) @@ -2429,7 +2486,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: CertUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newGroup.save().map(_ => Ok(newGroup.toJson)) } @@ -2458,6 +2515,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "UPDATE_CERTIFICATE", s"User updated a certificate", ctx.from, + ctx.ua, ctx.request.body ) Audit.send(event) @@ -2465,7 +2523,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: CertUpdatedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), - event) + event, ctx.from, ctx.ua) ) newGroup.save().map(_ => Ok(newGroup.toJson)) } @@ -2487,11 +2545,12 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "DELETE_CERTIFICATE", s"User deleted a certificate", ctx.from, + ctx.ua, Json.obj("CertId" -> CertId) ) Audit.send(event) Alerts.send( - CertDeleteAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event) + CertDeleteAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user.getOrElse(ctx.apiKey.toJson), event, ctx.from, ctx.ua) ) Ok(Json.obj("deleted" -> res)) } @@ -2509,6 +2568,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: env.env, Some(ctx.apiKey), ctx.user, + ctx.ua, "ACCESS_ALL_CERTIFICATES", s"User accessed all certificates", ctx.from @@ -2550,6 +2610,7 @@ class ApiController(ApiAction: ApiAction, UnAuthApiAction: UnAuthApiAction, cc: "ACCESS_CERTIFICATE", s"User accessed a certificate", ctx.from, + ctx.ua, Json.obj("certId" -> CertId) ) ) diff --git a/otoroshi/app/controllers/Auth0Controller.scala b/otoroshi/app/controllers/Auth0Controller.scala index a832f8db76..a17c78772b 100644 --- a/otoroshi/app/controllers/Auth0Controller.scala +++ b/otoroshi/app/controllers/Auth0Controller.scala @@ -8,19 +8,17 @@ import akka.http.scaladsl.util.FastFuture import akka.util.ByteString import auth.{AuthModuleConfig, BasicAuthModule, BasicAuthModuleConfig} import env.Env -import events.{AdminFirstLogin, AdminLoggedInAlert, AdminLoggedOutAlert, Alerts} +import events._ import gateway.Errors import models.{BackOfficeUser, CorsSettings, PrivateAppsUser, ServiceDescriptor} -import cluster.ClusterMode import play.api.Logger import play.api.libs.json.Json -import play.api.mvc.Results.BadRequest import play.api.mvc._ import security.IdGenerator import utils.TypedMap import utils.future.Implicits._ -import scala.concurrent.{ExecutionContext, Future} +import scala.concurrent.Future import scala.concurrent.duration.Duration class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, @@ -85,8 +83,8 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, } def confidentialAppLoginPage() = PrivateAppsAction.async { ctx => - import utils.future.Implicits._ import utils.RequestImplicits._ + import utils.future.Implicits._ implicit val req = ctx.request @@ -183,8 +181,8 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, } def confidentialAppCallback() = PrivateAppsAction.async { ctx => - import utils.future.Implicits._ import utils.RequestImplicits._ + import utils.future.Implicits._ implicit val req = ctx.request @@ -195,6 +193,7 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, .save(Duration(auth.sessionMaxAge, TimeUnit.SECONDS)) .map { paUser => env.clusterAgent.createSession(paUser) + Alerts.send(UserLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, paUser, ctx.from, ctx.ua)) ctx.request.session .get(s"pa-redirect-after-login-${auth.cookieSuffix(descriptor)}") .getOrElse( @@ -319,7 +318,7 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, ctx.user.simpleLogin match { case true => ctx.user.delete().map { _ => - Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user)) + Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, ctx.from, ctx.ua)) val userRedirect = redirect.getOrElse(routes.BackOfficeController.index().url) Redirect(userRedirect).removingFromSession("bousr", "bo-redirect-after-login") } @@ -328,7 +327,7 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, config.backOfficeAuthRef match { case None => { ctx.user.delete().map { _ => - Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user)) + Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, ctx.from, ctx.ua)) val userRedirect = redirect.getOrElse(routes.BackOfficeController.index().url) Redirect(userRedirect).removingFromSession("bousr", "bo-redirect-after-login") } @@ -344,7 +343,7 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, oauth.authModule(config).boLogout(ctx.request, config).flatMap { case None => { ctx.user.delete().map { _ => - Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user)) + Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, ctx.from, ctx.ua)) val userRedirect = redirect.getOrElse(routes.BackOfficeController.index().url) Redirect(userRedirect).removingFromSession("bousr", "bo-redirect-after-login") } @@ -353,7 +352,7 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, val userRedirect = redirect.getOrElse(s"http://${request.host}/") val actualRedirectUrl = logoutUrl.replace("${redirect}", URLEncoder.encode(userRedirect, "UTF-8")) ctx.user.delete().map { _ => - Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user)) + Alerts.send(AdminLoggedOutAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, ctx.from, ctx.ua)) Redirect(actualRedirectUrl).removingFromSession("bousr", "bo-redirect-after-login") } } @@ -377,11 +376,11 @@ class AuthController(BackOfficeActionAuth: BackOfficeActionAuth, env.datastores.backOfficeUserDataStore.hasAlreadyLoggedIn(user.email).map { case false => { env.datastores.backOfficeUserDataStore.alreadyLoggedIn(user.email) - Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } case true => { Alerts - .send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + .send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } } Redirect( diff --git a/otoroshi/app/controllers/BackOfficeController.scala b/otoroshi/app/controllers/BackOfficeController.scala index c272c9cece..b3127c62f4 100644 --- a/otoroshi/app/controllers/BackOfficeController.scala +++ b/otoroshi/app/controllers/BackOfficeController.scala @@ -261,6 +261,7 @@ class BackOfficeController(BackOfficeAction: BackOfficeAction, "SERVICESEARCH", "user searched for a service", ctx.from, + ctx.ua, Json.obj( "query" -> query )) @@ -356,10 +357,11 @@ class BackOfficeController(BackOfficeAction: BackOfficeAction, "DISCARD_SESSION", s"Admin discarded an Admin session", ctx.from, + ctx.ua, Json.obj("sessionId" -> id) ) Audit.send(event) - Alerts.send(SessionDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(SessionDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } recover { @@ -377,10 +379,11 @@ class BackOfficeController(BackOfficeAction: BackOfficeAction, "DISCARD_SESSIONS", s"Admin discarded Admin sessions", ctx.from, + ctx.ua, Json.obj() ) Audit.send(event) - Alerts.send(SessionsDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(SessionsDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } recover { @@ -408,10 +411,11 @@ class BackOfficeController(BackOfficeAction: BackOfficeAction, "DISCARD_PRIVATE_APPS_SESSION", s"Admin discarded a private app session", ctx.from, + ctx.ua, Json.obj("sessionId" -> id) ) Audit.send(event) - Alerts.send(SessionDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(SessionDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } recover { @@ -429,10 +433,11 @@ class BackOfficeController(BackOfficeAction: BackOfficeAction, "DISCARD_PRIVATE_APPS_SESSIONS", s"Admin discarded private apps sessions", ctx.from, + ctx.ua, Json.obj() ) Audit.send(event) - Alerts.send(SessionsDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(SessionsDiscardedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } recover { @@ -453,10 +458,11 @@ class BackOfficeController(BackOfficeAction: BackOfficeAction, "ACTIVATE_PANIC_MODE", s"Admin activated panic mode", ctx.from, + ctx.ua, Json.obj() ) Audit.send(event) - Alerts.send(PanicModeAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(PanicModeAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } recover { case _ => Ok(Json.obj("done" -> false)) diff --git a/otoroshi/app/controllers/U2FController.scala b/otoroshi/app/controllers/U2FController.scala index 42ec3c1d40..dd95c8a96a 100644 --- a/otoroshi/app/controllers/U2FController.scala +++ b/otoroshi/app/controllers/U2FController.scala @@ -86,10 +86,10 @@ class U2FController(BackOfficeAction: BackOfficeAction, env.datastores.simpleAdminDataStore.hasAlreadyLoggedIn(username).map { case false => { env.datastores.simpleAdminDataStore.alreadyLoggedIn(username) - Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } case true => { - Alerts.send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } } Ok(Json.obj("username" -> username)).addingToSession("bousr" -> boUser.randomId) @@ -140,10 +140,11 @@ class U2FController(BackOfficeAction: BackOfficeAction, "DELETE_ADMIN", s"Admin deleted an Admin", ctx.from, + ctx.ua, Json.obj("username" -> username) ) Audit.send(event) - Alerts.send(U2FAdminDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(U2FAdminDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } @@ -271,10 +272,10 @@ class U2FController(BackOfficeAction: BackOfficeAction, env.datastores.u2FAdminDataStore.hasAlreadyLoggedIn(username).map { case false => { env.datastores.u2FAdminDataStore.alreadyLoggedIn(username) - Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } case true => { - Alerts.send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } } Ok( @@ -315,10 +316,11 @@ class U2FController(BackOfficeAction: BackOfficeAction, "DELETE_U2F_ADMIN", s"Admin deleted an U2F Admin", ctx.from, + ctx.ua, Json.obj("username" -> username, "id" -> id) ) Audit.send(event) - Alerts.send(U2FAdminDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(U2FAdminDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } @@ -344,10 +346,11 @@ class U2FController(BackOfficeAction: BackOfficeAction, "DELETE_WEBAUTHN_ADMIN", s"Admin deleted a WebAuthn Admin", ctx.from, + ctx.ua, Json.obj("username" -> username, "id" -> id) ) Audit.send(event) - Alerts.send(WebAuthnAdminDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event)) + Alerts.send(WebAuthnAdminDeletedAlert(env.snowflakeGenerator.nextIdStr(), env.env, ctx.user, event, ctx.from, ctx.ua)) Ok(Json.obj("done" -> true)) } } @@ -589,10 +592,10 @@ class U2FController(BackOfficeAction: BackOfficeAction, env.datastores.u2FAdminDataStore.hasAlreadyLoggedIn(username).map { case false => { env.datastores.u2FAdminDataStore.alreadyLoggedIn(username) - Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminFirstLogin(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } case true => { - Alerts.send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser)) + Alerts.send(AdminLoggedInAlert(env.snowflakeGenerator.nextIdStr(), env.env, boUser, ctx.from, ctx.ua)) } } Ok( diff --git a/otoroshi/app/el/el.scala b/otoroshi/app/el/el.scala index 6c1d4bdcda..d50bc6c1be 100644 --- a/otoroshi/app/el/el.scala +++ b/otoroshi/app/el/el.scala @@ -24,10 +24,13 @@ object GlobalExpressionLanguage { apiKey: Option[ApiKey], user: Option[PrivateAppsUser], context: Map[String, String], + attrs: utils.TypedMap ): String = { // println(s"${req}:${service}:${apiKey}:${user}:${context}") value match { case v if v.contains("${") => + val userAgentDetails = attrs.get(otoroshi.plugins.Keys.UserAgentInfoKey) + val geolocDetails = attrs.get(otoroshi.plugins.Keys.GeolocationInfoKey) Try { expressionReplacer.replaceOn(value) { case "date" => DateTime.now().toString() @@ -100,7 +103,20 @@ object GlobalExpressionLanguage { context.get(field).orElse(context.get(field2)).getOrElse(s"no-ctx-$field-$field2") case r"ctx.$field@(.*):$dv@(.*)" => context.getOrElse(field, dv) case r"ctx.$field@(.*)" => context.getOrElse(field, s"no-ctx-$field") - + case r"ctx.useragent.$field@(.*)" if userAgentDetails.isDefined => + val lookup: JsLookupResult = (userAgentDetails.get.\(field)) + lookup.asOpt[String] + .orElse(lookup.asOpt[Long].map(_.toString)) + .orElse(lookup.asOpt[Double].map(_.toString)) + .orElse(lookup.asOpt[Boolean].map(_.toString)) + .getOrElse(s"no-ctx-$field") + case r"ctx.geolocation.$field@(.*)" if geolocDetails.isDefined => + val lookup: JsLookupResult = (geolocDetails.get.\(field)) + lookup.asOpt[String] + .orElse(lookup.asOpt[Long].map(_.toString)) + .orElse(lookup.asOpt[Double].map(_.toString)) + .orElse(lookup.asOpt[Boolean].map(_.toString)) + .getOrElse(s"no-ctx-$field") case "user.name" if user.isDefined => user.get.name case "user.email" if user.isDefined => user.get.email case r"user.metadata.$field@(.*):$dv@(.*)" if user.isDefined => @@ -154,6 +170,7 @@ object HeadersExpressionLanguage { apiKey: Option[ApiKey], user: Option[PrivateAppsUser], context: Map[String, String], + attrs: utils.TypedMap ): String = { GlobalExpressionLanguage.apply( value = value, @@ -161,7 +178,8 @@ object HeadersExpressionLanguage { service = service, apiKey = apiKey, user = user, - context = context + context = context, + attrs = attrs ) } @@ -231,6 +249,7 @@ object RedirectionExpressionLanguage { apiKey: Option[ApiKey], user: Option[PrivateAppsUser], context: Map[String, String], + attrs: utils.TypedMap ): String = { GlobalExpressionLanguage.apply( value = value, @@ -238,7 +257,8 @@ object RedirectionExpressionLanguage { service = service, apiKey = apiKey, user = user, - context = context + context = context, + attrs = attrs ) } @@ -285,6 +305,7 @@ object TargetExpressionLanguage { apiKey: Option[ApiKey], user: Option[PrivateAppsUser], context: Map[String, String], + attrs: utils.TypedMap ): String = { GlobalExpressionLanguage.apply( value = value, @@ -292,7 +313,8 @@ object TargetExpressionLanguage { service = service, apiKey = apiKey, user = user, - context = context + context = context, + attrs = attrs ) } @@ -335,7 +357,8 @@ object JwtExpressionLanguage { service: Option[ServiceDescriptor], apiKey: Option[ApiKey], user: Option[PrivateAppsUser], - context: Map[String, String] + context: Map[String, String], + attrs: utils.TypedMap ): String = { GlobalExpressionLanguage.apply( value = value, @@ -343,7 +366,8 @@ object JwtExpressionLanguage { service = service, apiKey = apiKey, user = user, - context = context + context = context, + attrs = attrs ) } @@ -388,25 +412,26 @@ object JwtExpressionLanguage { service: Option[ServiceDescriptor], apiKey: Option[ApiKey], user: Option[PrivateAppsUser], - context: Map[String, String] + context: Map[String, String], + attrs: utils.TypedMap ): JsValue = { value match { case JsObject(map) => new JsObject(map.toSeq.map { - case (key, JsString(str)) => (key, JsString(apply(str, req, service, apiKey, user, context))) - case (key, obj @ JsObject(_)) => (key, fromJson(obj, req, service, apiKey, user, context)) - case (key, arr @ JsArray(_)) => (key, fromJson(arr, req, service, apiKey, user, context)) + case (key, JsString(str)) => (key, JsString(apply(str, req, service, apiKey, user, context, attrs))) + case (key, obj @ JsObject(_)) => (key, fromJson(obj, req, service, apiKey, user, context, attrs)) + case (key, arr @ JsArray(_)) => (key, fromJson(arr, req, service, apiKey, user, context, attrs)) case (key, v) => (key, v) }.toMap) case JsArray(values) => new JsArray(values.map { - case JsString(str) => JsString(apply(str, req, service, apiKey, user, context)) - case obj: JsObject => fromJson(obj, req, service, apiKey, user, context) - case arr: JsArray => fromJson(arr, req, service, apiKey, user, context) + case JsString(str) => JsString(apply(str, req, service, apiKey, user, context, attrs)) + case obj: JsObject => fromJson(obj, req, service, apiKey, user, context, attrs) + case arr: JsArray => fromJson(arr, req, service, apiKey, user, context, attrs) case v => v }) case JsString(str) => { - apply(str, req, service, apiKey, user, context) match { + apply(str, req, service, apiKey, user, context, attrs) match { case "true" => JsBoolean(true) case "false" => JsBoolean(false) case r"$nbr@([0-9\\.,]+)" => JsNumber(nbr.toDouble) diff --git a/otoroshi/app/events/alerts.scala b/otoroshi/app/events/alerts.scala index 0558e8564f..2b96be4e29 100644 --- a/otoroshi/app/events/alerts.scala +++ b/otoroshi/app/events/alerts.scala @@ -38,6 +38,9 @@ case class MaxConcurrentRequestReachedAlert(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -63,6 +66,9 @@ case class HighOverheadAlert(`@id`: String, override def `@service`: String = serviceDescriptor.name override def `@serviceId`: String = serviceDescriptor.id + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -88,6 +94,9 @@ case class CircuitBreakerOpenedAlert(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = service.id + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -112,6 +121,9 @@ case class CircuitBreakerClosedAlert(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = service.id + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -130,12 +142,17 @@ case class SessionDiscardedAlert(`@id`: String, `@env`: String, user: BackOfficeUser, event: BackOfficeEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -153,12 +170,17 @@ case class SessionsDiscardedAlert(`@id`: String, `@env`: String, user: BackOfficeUser, event: BackOfficeEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -176,12 +198,17 @@ case class PanicModeAlert(`@id`: String, `@env`: String, user: BackOfficeUser, event: BackOfficeEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -200,12 +227,17 @@ case class OtoroshiExportAlert(`@id`: String, user: JsValue, event: AdminApiEvent, export: JsValue, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -225,10 +257,14 @@ case class SnowMonkeyStartedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -247,10 +283,14 @@ case class SnowMonkeyStoppedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -269,10 +309,14 @@ case class SnowMonkeyConfigUpdatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -291,10 +335,14 @@ case class SnowMonkeyResetAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -314,10 +362,14 @@ case class CertCreatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -337,10 +389,14 @@ case class CertUpdatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -360,10 +416,14 @@ case class CertDeleteAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -386,6 +446,8 @@ case class SnowMonkeyOutageRegisteredAlert(`@id`: String, extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -405,12 +467,17 @@ case class U2FAdminDeletedAlert(`@id`: String, `@env`: String, user: BackOfficeUser, event: BackOfficeEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -429,12 +496,17 @@ case class WebAuthnAdminDeletedAlert(`@id`: String, `@env`: String, user: BackOfficeUser, event: BackOfficeEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -452,12 +524,17 @@ case class WebAuthnAdminDeletedAlert(`@id`: String, case class BlackListedBackOfficeUserAlert(`@id`: String, `@env`: String, user: BackOfficeUser, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -474,12 +551,17 @@ case class BlackListedBackOfficeUserAlert(`@id`: String, case class AdminLoggedInAlert(`@id`: String, `@env`: String, user: BackOfficeUser, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -496,12 +578,45 @@ case class AdminLoggedInAlert(`@id`: String, ) } -case class AdminFirstLogin(`@id`: String, `@env`: String, user: BackOfficeUser, `@timestamp`: DateTime = DateTime.now()) +case class UserLoggedInAlert(`@id`: String, + `@env`: String, + user: PrivateAppsUser, + from: String, + ua: String, + `@timestamp`: DateTime = DateTime.now()) + extends AlertEvent { + + override def `@service`: String = "Otoroshi" + override def `@serviceId`: String = "--" + + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + + override def toJson(implicit _env: Env): JsValue = Json.obj( + "@id" -> `@id`, + "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), + "@type" -> `@type`, + "@product" -> _env.eventsName, + "@serviceId" -> `@serviceId`, + "@service" -> `@service`, + "@env" -> `@env`, + "alert" -> "UserLoggedInAlert", + "userName" -> user.name, + "userEmail" -> user.email, + "user" -> user.profile, + "userRandomId" -> user.randomId + ) +} + +case class AdminFirstLogin(`@id`: String, `@env`: String, user: BackOfficeUser, from: String, ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -521,12 +636,17 @@ case class AdminFirstLogin(`@id`: String, `@env`: String, user: BackOfficeUser, case class AdminLoggedOutAlert(`@id`: String, `@env`: String, user: BackOfficeUser, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -543,12 +663,15 @@ case class AdminLoggedOutAlert(`@id`: String, ) } -case class DbResetAlert(`@id`: String, `@env`: String, user: JsValue, `@timestamp`: DateTime = DateTime.now()) +case class DbResetAlert(`@id`: String, `@env`: String, user: JsValue, from: String, ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -562,12 +685,15 @@ case class DbResetAlert(`@id`: String, `@env`: String, user: JsValue, `@timestam ) } -case class DangerZoneAccessAlert(`@id`: String, `@env`: String, user: JsValue, `@timestamp`: DateTime = DateTime.now()) +case class DangerZoneAccessAlert(`@id`: String, `@env`: String, user: JsValue, from: String, ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -587,12 +713,17 @@ case class GlobalConfigModification(`@id`: String, oldConfig: JsValue, newConfig: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -621,6 +752,9 @@ case class RevokedApiKeyUsageAlert(`@id`: String, override def `@service`: String = descriptor.name override def `@serviceId`: String = descriptor.id + override def fromOrigin: Option[String] = Some(req.headers.get("X-Forwarded-For").getOrElse(req.remoteAddress)) + override def fromUserAgent: Option[String] = Some(req.headers.get("User-Agent").getOrElse("none")) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -640,10 +774,14 @@ case class ServiceGroupCreatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -662,10 +800,14 @@ case class ServiceGroupUpdatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -684,10 +826,14 @@ case class ServiceGroupDeletedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -706,10 +852,14 @@ case class ServiceCreatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -728,10 +878,14 @@ case class ServiceUpdatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -750,10 +904,14 @@ case class ServiceDeletedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -772,10 +930,14 @@ case class ApiKeyCreatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -794,10 +956,14 @@ case class ApiKeyUpdatedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -816,10 +982,14 @@ case class ApiKeyDeletedAlert(`@id`: String, `@env`: String, user: JsValue, audit: AuditEvent, + from: String, + ua: String, `@timestamp`: DateTime = DateTime.now()) extends AlertEvent { override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -854,10 +1024,10 @@ class AlertsActor(implicit env: Env) extends Actor { lazy val emailStream = Source .queue[AlertEvent](5000, OverflowStrategy.dropHead) + .mapAsync(5)(evt => evt.toEnrichedJson) .groupedWithin(25, FiniteDuration(60, TimeUnit.SECONDS)) .mapAsync(1) { evts => val titles = evts - .map(_.toEnrichedJson) .map { jsonEvt => val date = new DateTime((jsonEvt \ "@timestamp").as[Long]) val id = (jsonEvt \ "@id").as[String] @@ -868,7 +1038,6 @@ class AlertsActor(implicit env: Env) extends Actor { .mkString("") val email = evts - .map(_.toEnrichedJson) .map { jsonEvt => val alert = (jsonEvt \ "alert").asOpt[String].getOrElse("Unkown alert") val message = (jsonEvt \ "audit" \ "message").asOpt[String].getOrElse("No description message") @@ -900,11 +1069,13 @@ class AlertsActor(implicit env: Env) extends Actor { ) } - lazy val stream = Source.queue[AlertEvent](50000, OverflowStrategy.dropHead).mapAsync(5) { evt => + lazy val stream = Source.queue[AlertEvent](50000, OverflowStrategy.dropHead).mapAsync(5) { _evt => for { - r <- env.datastores.globalConfigDataStore.singleton().flatMap { config => + evt <- _evt.toEnrichedJson + config <- env.datastores.globalConfigDataStore.singleton() + r <- { config.kafkaConfig.foreach { kafkaConfig => - kafkaWrapper.publish(evt.toEnrichedJson)(env, kafkaConfig) + kafkaWrapper.publish(evt)(env, kafkaConfig) } if (config.kafkaConfig.isEmpty) kafkaWrapper.close() Future.sequence(config.alertsWebhooks.map { webhook => @@ -913,7 +1084,7 @@ class AlertsActor(implicit env: Env) extends Actor { .url(url) .withHttpHeaders(webhook.headers.toSeq: _*) .withMaybeProxyServer(config.proxies.alertWebhooks) - .post(Json.obj("event" -> "ALERT", "payload" -> evt.toEnrichedJson)) + .post(Json.obj("event" -> "ALERT", "payload" -> evt)) .andThen { case Success(r) => r.ignore() case Failure(e) => { @@ -993,7 +1164,7 @@ object Alerts { lazy val logger = Logger("otoroshi-alerts") def send[A <: AlertEvent](alert: A)(implicit env: Env): Unit = { - logger.trace("Alert " + Json.stringify(alert.toEnrichedJson)) + // logger.trace("Alert " + Json.stringify(alert.toEnrichedJson)) alert.toAnalytics() env.alertsActor ! alert } @@ -1002,5 +1173,6 @@ object Alerts { trait AlertDataStore { def count()(implicit ec: ExecutionContext, env: Env): Future[Long] def findAllRaw(from: Long = 0, to: Long = 1000)(implicit ec: ExecutionContext, env: Env): Future[Seq[ByteString]] - def push(event: AlertEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] + // def push(event: AlertEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] + def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] } diff --git a/otoroshi/app/events/analytics.scala b/otoroshi/app/events/analytics.scala index 1b940af84b..06a144751b 100644 --- a/otoroshi/app/events/analytics.scala +++ b/otoroshi/app/events/analytics.scala @@ -13,6 +13,7 @@ import env.Env import events.impl.{ElasticReadsAnalytics, ElasticWritesAnalytics, WebHookAnalytics} import models._ import org.joda.time.DateTime +import otoroshi.plugins.useragent.UserAgentHelper import otoroshi.tcp.TcpService import play.api.Logger import play.api.libs.json._ @@ -39,6 +40,7 @@ class AnalyticsActor(implicit env: Env) extends Actor { lazy val stream = Source .queue[AnalyticEvent](50000, OverflowStrategy.dropHead) + .mapAsync(5)(evt => evt.toEnrichedJson) .groupedWithin(env.maxWebhookSize, FiniteDuration(env.analyticsWindow, TimeUnit.SECONDS)) .mapAsync(5) { evts => logger.debug(s"SEND_TO_ANALYTICS_HOOK: will send ${evts.size} evts") @@ -46,8 +48,8 @@ class AnalyticsActor(implicit env: Env) extends Actor { logger.debug("SEND_TO_ANALYTICS_HOOK: " + config.analyticsWebhooks) config.kafkaConfig.foreach { kafkaConfig => evts.foreach { - case evt: AuditEvent => kafkaWrapperAudit.publish(evt.toEnrichedJson)(env, kafkaConfig) - case evt => kafkaWrapperAnalytics.publish(evt.toEnrichedJson)(env, kafkaConfig) + case evt: AuditEvent => kafkaWrapperAudit.publish(evt)(env, kafkaConfig) + case evt => kafkaWrapperAnalytics.publish(evt)(env, kafkaConfig) } if (config.kafkaConfig.isEmpty) { kafkaWrapperAnalytics.close() @@ -136,10 +138,34 @@ trait AnalyticEvent { def `@timestamp`: DateTime def `@service`: String def `@serviceId`: String + def fromOrigin: Option[String] + def fromUserAgent: Option[String] def toJson(implicit _env: Env): JsValue - def toEnrichedJson(implicit _env: Env): JsValue = { - toJson(_env).as[JsObject] ++ Json.obj( + def toEnrichedJson(implicit _env: Env, ec: ExecutionContext): Future[JsValue] = { + val uaDetails = fromUserAgent match { + case None => JsNull + case Some(ua) => UserAgentHelper.userAgentDetails(ua) match { + case None => JsNull + case Some(details) => details + } + } + val fOrigin = fromOrigin match { + case None => FastFuture.successful(JsNull) + case Some(ipAddress) => { + _env.datastores.globalConfigDataStore.latestSafe match { + case None => FastFuture.successful(JsNull) + case Some(config) if !config.geolocationSettings.enabled => FastFuture.successful(JsNull) + case Some(config) => config.geolocationSettings.find(ipAddress).map { + case None => JsNull + case Some(details) => details + } + } + } + } + fOrigin.map(originDetails => toJson(_env).as[JsObject] ++ Json.obj( + "user-agent-details" -> uaDetails, + "origin-details" -> originDetails, "instance-name" -> _env.name, "instance-zone" -> _env.zone, "instance-region" -> _env.region, @@ -152,7 +178,7 @@ trait AnalyticEvent { case ClusterMode.Leader => _env.clusterConfig.leader.name case _ => "none" }) - ) + )) } def toAnalytics()(implicit env: Env): Unit = { @@ -160,8 +186,8 @@ trait AnalyticEvent { // Logger("otoroshi-analytics").debug(s"${this.`@type`} ${Json.stringify(toJson)}") } - def log()(implicit _env: Env): Unit = { - AnalyticEvent.logger.info(Json.stringify(toEnrichedJson)) + def log()(implicit _env: Env, ec: ExecutionContext): Unit = { + toEnrichedJson.map(e => AnalyticEvent.logger.info(Json.stringify(e))) } } @@ -235,6 +261,8 @@ case class GatewayEvent( userAgentInfo: Option[JsValue], geolocationInfo: Option[JsValue], ) extends AnalyticEvent { + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = headers.find(h => h.key.toLowerCase() == "user-agent").map(_.value) def toJson(implicit _env: Env): JsValue = GatewayEvent.writes(this, _env) } @@ -297,6 +325,8 @@ case class TcpEvent( `@service`: String, service: Option[TcpService], ) extends AnalyticEvent { + override def fromOrigin: Option[String] = Some(remote) + override def fromUserAgent: Option[String] = None def toJson(implicit _env: Env): JsValue = TcpEvent.writes(this, _env) } @@ -336,9 +366,10 @@ case class HealthCheckEvent( error: Option[String] = None, health: Option[String] = None ) extends AnalyticEvent { + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None def toJson(implicit _env: Env): JsValue = HealthCheckEvent.format.writes(this) - def pushToRedis()(implicit ec: ExecutionContext, env: Env): Future[Long] = - env.datastores.healthCheckDataStore.push(this) + def pushToRedis()(implicit ec: ExecutionContext, env: Env): Future[Long] = toEnrichedJson.flatMap(e => env.datastores.healthCheckDataStore.push(e)) def isUp: Boolean = if (error.isDefined) { false @@ -360,7 +391,7 @@ trait HealthCheckDataStore { env: Env): Future[Seq[HealthCheckEvent]] def findLast(serviceDescriptor: ServiceDescriptor)(implicit ec: ExecutionContext, env: Env): Future[Option[HealthCheckEvent]] - def push(event: HealthCheckEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] + def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] } sealed trait Filterable @@ -448,7 +479,7 @@ trait AnalyticsReadsService { trait AnalyticsWritesService { def init(): Unit - def publish(event: Seq[AnalyticEvent])(implicit env: Env, ec: ExecutionContext): Future[Unit] + def publish(event: Seq[JsValue])(implicit env: Env, ec: ExecutionContext): Future[Unit] } class AnalyticsReadsServiceImpl(globalConfig: GlobalConfig, env: Env) extends AnalyticsReadsService { diff --git a/otoroshi/app/events/audit.scala b/otoroshi/app/events/audit.scala index 3a5cc007fe..04121d3eaf 100644 --- a/otoroshi/app/events/audit.scala +++ b/otoroshi/app/events/audit.scala @@ -19,6 +19,7 @@ case class BackOfficeEvent(`@id`: String, action: String, message: String, from: String, + ua: String, metadata: JsObject = Json.obj(), `@timestamp`: DateTime = DateTime.now()) extends AuditEvent { @@ -26,6 +27,9 @@ case class BackOfficeEvent(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -53,6 +57,7 @@ case class AdminApiEvent(`@id`: String, action: String, message: String, from: String, + ua: String, metadata: JsValue = Json.obj(), `@timestamp`: DateTime = DateTime.now()) extends AuditEvent { @@ -60,6 +65,9 @@ case class AdminApiEvent(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -91,6 +99,9 @@ case class SnowMonkeyOutageRegisteredEvent(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -121,6 +132,9 @@ case class CircuitBreakerOpenedEvent(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = service.id + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -145,6 +159,9 @@ case class CircuitBreakerClosedEvent(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = service.id + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -169,6 +186,9 @@ case class MaxConcurrentRequestReachedEvent(`@id`: String, override def `@service`: String = "Otoroshi" override def `@serviceId`: String = "--" + override def fromOrigin: Option[String] = None + override def fromUserAgent: Option[String] = None + override def toJson(implicit _env: Env): JsValue = Json.obj( "@id" -> `@id`, "@timestamp" -> play.api.libs.json.JodaWrites.JodaDateTimeNumberWrites.writes(`@timestamp`), @@ -187,12 +207,12 @@ object Audit { def send[A <: AuditEvent](audit: A)(implicit env: Env): Unit = { implicit val ec = env.otoroshiExecutionContext audit.toAnalytics() - env.datastores.auditDataStore.push(audit) + audit.toEnrichedJson.map(e => env.datastores.auditDataStore.push(e)) } } trait AuditDataStore { def count()(implicit ec: ExecutionContext, env: Env): Future[Long] def findAllRaw(from: Long = 0, to: Long = 1000)(implicit ec: ExecutionContext, env: Env): Future[Seq[ByteString]] - def push(event: AuditEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] + def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] } diff --git a/otoroshi/app/events/impl/ElasticAnalytics.scala b/otoroshi/app/events/impl/ElasticAnalytics.scala index 5175e96ceb..efd06855b1 100644 --- a/otoroshi/app/events/impl/ElasticAnalytics.scala +++ b/otoroshi/app/events/impl/ElasticAnalytics.scala @@ -216,7 +216,7 @@ class ElasticWritesAnalytics(config: ElasticAnalyticsConfig, } yield s"Basic ${Base64.getEncoder.encodeToString(s"$user:$password".getBytes())}" } - override def publish(event: Seq[AnalyticEvent])(implicit env: Env, ec: ExecutionContext): Future[Unit] = { + override def publish(event: Seq[JsValue])(implicit env: Env, ec: ExecutionContext): Future[Unit] = { val builder = env.Ws .url(urlFromPath("/_bulk")) .withMaybeProxyServer(env.datastores.globalConfigDataStore.latestSafe.flatMap(_.proxies.elastic)) @@ -234,7 +234,6 @@ class ElasticWritesAnalytics(config: ElasticAnalyticsConfig, } .addHttpHeaders(config.headers.toSeq: _*) Source(event.toList) - .map(_.toEnrichedJson) .grouped(500) .map(_.map(bulkRequest)) .mapAsync(10) { bulk => diff --git a/otoroshi/app/events/impl/WebHookAnalytics.scala b/otoroshi/app/events/impl/WebHookAnalytics.scala index c9572d1239..c4d69dc20e 100644 --- a/otoroshi/app/events/impl/WebHookAnalytics.scala +++ b/otoroshi/app/events/impl/WebHookAnalytics.scala @@ -52,7 +52,7 @@ class WebHookAnalytics(webhook: Webhook, config: GlobalConfig) extends Analytics ) ).flatten - override def publish(event: Seq[AnalyticEvent])(implicit env: Env, ec: ExecutionContext): Future[Unit] = { + override def publish(event: Seq[JsValue])(implicit env: Env, ec: ExecutionContext): Future[Unit] = { val state = IdGenerator.extendedToken(128) val claim = OtoroshiClaim( iss = env.Headers.OtoroshiIssuer, @@ -72,17 +72,17 @@ class WebHookAnalytics(webhook: Webhook, config: GlobalConfig) extends Analytics evt => webhook.url //.replace("@product", env.eventsName) - .replace("@service", evt.`@service`) - .replace("@serviceId", evt.`@serviceId`) - .replace("@id", evt.`@id`) - .replace("@messageType", evt.`@type`) + .replace("@service", (evt \ "@service").as[String]) + .replace("@serviceId", (evt \ "@serviceId").as[String]) + .replace("@id", (evt \ "@id").as[String]) + .replace("@messageType", (evt \ "@type").as[String]) ) .getOrElse(webhook.url) val postResponse = env.Ws .url(url) .withHttpHeaders(headers: _*) .withMaybeProxyServer(config.proxies.eventsWebhooks) - .post(JsArray(event.map(_.toEnrichedJson))) + .post(JsArray(event)) postResponse.andThen { case Success(resp) => { logger.debug(s"SEND_TO_ANALYTICS_SUCCESS: ${resp.status} - ${resp.headers} - ${resp.body}") diff --git a/otoroshi/app/gateway/handlers.scala b/otoroshi/app/gateway/handlers.scala index 36cdb1540f..99c57697cf 100644 --- a/otoroshi/app/gateway/handlers.scala +++ b/otoroshi/app/gateway/handlers.scala @@ -672,7 +672,7 @@ class GatewayRequestHandler(snowMonkey: SnowMonkey, f } else { val inputHeaders = req.headers.toSimpleMap - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(desc), apiKey, paUsr, ctx)) + .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(desc), apiKey, paUsr, ctx, attrs)) .filterNot(h => h._2 == "null") desc.headersVerification.map(tuple => inputHeaders.get(tuple._1).exists(_ == tuple._2)).find(_ == false) match { case Some(_) => @@ -891,7 +891,7 @@ class GatewayRequestHandler(snowMonkey: SnowMonkey, FastFuture.successful( Results .Status(rawDesc.redirection.code) - .withHeaders("Location" -> rawDesc.redirection.formattedTo(req, rawDesc, elCtx)) + .withHeaders("Location" -> rawDesc.redirection.formattedTo(req, rawDesc, elCtx, attrs)) ) } case Some(rawDesc) => { @@ -1204,14 +1204,14 @@ class GatewayRequestHandler(snowMonkey: SnowMonkey, Some(descriptor), apiKey, paUsr, - elCtx) + elCtx, attrs) val root = descriptor.root val url = TargetExpressionLanguage(s"$scheme://$host$root$uri", Some(req), Some(descriptor), apiKey, paUsr, - elCtx) + elCtx, attrs) lazy val currentReqHasBody = hasBody(req) // val queryString = req.queryString.toSeq.flatMap { case (key, values) => values.map(v => (key, v)) } val fromOtoroshi = req.headers @@ -1240,71 +1240,9 @@ class GatewayRequestHandler(snowMonkey: SnowMonkey, stateToken = stateToken, fromOtoroshi = fromOtoroshi, snowMonkeyContext = snowMonkeyContext, - jwtInjection = jwtInjection + jwtInjection = jwtInjection, + attrs = attrs ) - //val stateRequestHeaderName = - // descriptor.secComHeaders.stateRequestName.getOrElse(env.Headers.OtoroshiState) - //val claimRequestHeaderName = - // descriptor.secComHeaders.claimRequestName.getOrElse(env.Headers.OtoroshiClaim) - /* - val headersIn: Seq[(String, String)] = { - (desc.missingOnlyHeadersIn.filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)).filterNot(h => h._2 == "null") ++ - req.headers.toMap.toSeq - .flatMap(c => c._2.map(v => (c._1, v))) //.map(tuple => (tuple._1, tuple._2.mkString(","))) //.toSimpleMap - .filterNot( - t => - if (t._1.toLowerCase == "content-type" && !currentReqHasBody) true - else if (t._1.toLowerCase == "content-length") true - else false - ) - .filterNot(t => descriptor.removeHeadersIn.contains(t._1)) - .filterNot( - t => - (headersInFiltered ++ Seq(stateRequestHeaderName, claimRequestHeaderName)) - .contains(t._1.toLowerCase) - ) ++ Map( - env.Headers.OtoroshiProxiedHost -> req.headers.get("Host").getOrElse("--"), - //"Host" -> host, - "Host" -> (if (desc.overrideHost) host - else req.headers.get("Host").getOrElse("--")), - env.Headers.OtoroshiRequestId -> snowflake, - env.Headers.OtoroshiRequestTimestamp -> requestTimestamp - ) ++ (if (descriptor.enforceSecureCommunication && descriptor.sendInfoToken) { - Map( - claimRequestHeaderName -> claim - ) - } else { - Map.empty[String, String] - }) ++ (if (descriptor.enforceSecureCommunication && descriptor.sendStateChallenge) { - Map( - stateRequestHeaderName -> stateToken - ) - } else { - Map.empty[String, String] - }) ++ (req.clientCertificateChain match { - case Some(chain) => - Map(env.Headers.OtoroshiClientCertChain -> req.clientCertChainPemString) - case None => Map.empty[String, String] - }) ++ req.headers - .get("Content-Length") - .map(l => { - Map( - "Content-Length" -> (l.toInt + snowMonkeyContext.trailingRequestBodySize).toString - ) - }) - .getOrElse(Map.empty[String, String]) ++ - descriptor.additionalHeaders - .filter(t => t._1.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)).filterNot(h => h._2 == "null") ++ fromOtoroshi - .map(v => Map(env.Headers.OtoroshiGatewayParentRequest -> fromOtoroshi.get)) - .getOrElse(Map.empty[String, String]) ++ jwtInjection.additionalHeaders).toSeq - .filterNot(t => jwtInjection.removeHeaders.contains(t._1)) ++ xForwardedHeader( - desc, - req - ) - } - */ val lazySource = Source.single(ByteString.empty).flatMapConcat { _ => bodyAlreadyConsumed.compareAndSet(false, true) @@ -1535,46 +1473,9 @@ class GatewayRequestHandler(snowMonkey: SnowMonkey, overhead = overhead, upstreamLatency = 0L, canaryId = canaryId, - remainingQuotas = remainingQuotas + remainingQuotas = remainingQuotas, + attrs = attrs ) - // val _headersOut: Seq[(String, String)] = { - // descriptor.missingOnlyHeadersOut.filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - // .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)).filterNot(h => h._2 == "null").toSeq ++ - // badResult.header.headers.toSeq - // .filterNot(t => descriptor.removeHeadersOut.contains(t._1)) - // .filterNot( - // t => - // (headersOutFiltered :+ stateResponseHeaderName) - // .contains(t._1.toLowerCase) - // ) ++ ( - // if (descriptor.sendOtoroshiHeadersBack) { - // Seq( - // env.Headers.OtoroshiRequestId -> snowflake, - // env.Headers.OtoroshiRequestTimestamp -> requestTimestamp, - // env.Headers.OtoroshiProxyLatency -> s"$overhead", - // env.Headers.OtoroshiUpstreamLatency -> s"0" - // ) - // } else { - // Seq.empty[(String, String)] - // } - // ) ++ Some(canaryId) - // .filter(_ => desc.canary.enabled) - // .map( - // _ => - // env.Headers.OtoroshiTrackerId -> s"${env.sign(canaryId)}::$canaryId" - // ) ++ (if (descriptor.sendOtoroshiHeadersBack && apiKey.isDefined) { - // Seq( - // env.Headers.OtoroshiDailyCallsRemaining -> remainingQuotas.remainingCallsPerDay.toString, - // env.Headers.OtoroshiMonthlyCallsRemaining -> remainingQuotas.remainingCallsPerMonth.toString - // ) - // } else { - // Seq.empty[(String, String)] - // }) ++ descriptor.cors.asHeaders(req) ++ desc.additionalHeadersOut - // .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - // .mapValues( - // v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx) - // ).filterNot(h => h._2 == "null").toSeq - // } promise.trySuccess( ProxyDone( badResult.header.status, @@ -1767,43 +1668,9 @@ class GatewayRequestHandler(snowMonkey: SnowMonkey, overhead = overhead, upstreamLatency = upstreamLatency, canaryId = canaryId, - remainingQuotas = remainingQuotas + remainingQuotas = remainingQuotas, + attrs = attrs ) - // val _headersOut: Seq[(String, String)] = { - // descriptor.missingOnlyHeadersOut.filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - // .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)).filterNot(h => h._2 == "null").toSeq ++ - // _headersForOut - // .filterNot(t => descriptor.removeHeadersOut.contains(t._1)) - // .filterNot(t => headersOutFiltered.contains(t._1.toLowerCase)) ++ ( - // if (descriptor.sendOtoroshiHeadersBack) { - // Seq( - // env.Headers.OtoroshiRequestId -> snowflake, - // env.Headers.OtoroshiRequestTimestamp -> requestTimestamp, - // env.Headers.OtoroshiProxyLatency -> s"$overhead", - // env.Headers.OtoroshiUpstreamLatency -> s"$upstreamLatency" //, - // //env.Headers.OtoroshiTrackerId -> s"${env.sign(trackingId)}::$trackingId" - // ) - // } else { - // Seq.empty[(String, String)] - // } - // ) ++ Some(canaryId) - // .filter(_ => desc.canary.enabled) - // .map( - // _ => env.Headers.OtoroshiTrackerId -> s"${env.sign(canaryId)}::$canaryId" - // ) ++ (if (descriptor.sendOtoroshiHeadersBack && apiKey.isDefined) { - // Seq( - // env.Headers.OtoroshiDailyCallsRemaining -> remainingQuotas.remainingCallsPerDay.toString, - // env.Headers.OtoroshiMonthlyCallsRemaining -> remainingQuotas.remainingCallsPerMonth.toString - // ) - // } else { - // Seq.empty[(String, String)] - // }) ++ descriptor.cors - // .asHeaders(req) ++ desc.additionalHeadersOut - // .mapValues( - // v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx) - // ).filterNot(h => h._2 == "null") - // .toSeq - // } val otoroshiResponse = otoroshi.script.HttpResponse( status = resp.status, diff --git a/otoroshi/app/gateway/websockets.scala b/otoroshi/app/gateway/websockets.scala index f85ac56fbf..bff8f216f1 100644 --- a/otoroshi/app/gateway/websockets.scala +++ b/otoroshi/app/gateway/websockets.scala @@ -411,7 +411,7 @@ class WebSocketHandler()(implicit env: Env) { .successful( Results .Status(rawDesc.redirection.code) - .withHeaders("Location" -> rawDesc.redirection.formattedTo(req, rawDesc, elCtx)) + .withHeaders("Location" -> rawDesc.redirection.formattedTo(req, rawDesc, elCtx, attrs)) ) .asLeft[WSFlow] } @@ -668,7 +668,8 @@ class WebSocketHandler()(implicit env: Env) { Some(descriptor), apiKey, paUsr, - elCtx + elCtx, + attrs ) // val queryString = req.queryString.toSeq.flatMap { case (key, values) => values.map(v => (key, v)) } val fromOtoroshi = req.headers @@ -701,7 +702,8 @@ class WebSocketHandler()(implicit env: Env) { Source.empty[ByteString], Source.empty[ByteString] ), - jwtInjection = jwtInjection + jwtInjection = jwtInjection, + attrs = attrs ) //val claimRequestHeaderName = // descriptor.secComHeaders.claimRequestName.getOrElse(env.Headers.OtoroshiClaim) @@ -942,7 +944,7 @@ class WebSocketHandler()(implicit env: Env) { .mapValues( v => HeadersExpressionLanguage - .apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx) + .apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs) ) .filterNot(h => h._2 == "null") .toSeq @@ -980,7 +982,7 @@ class WebSocketHandler()(implicit env: Env) { Some(descriptor), apiKey, paUsr, - elCtx)) match { + elCtx, attrs)) match { case (_, host) if host.contains(":") => (host.split(":").apply(0), host.split(":").apply(1).toInt) case (scheme, host) if scheme.contains("https") => (host, 443) diff --git a/otoroshi/app/models/JWTVerifier.scala b/otoroshi/app/models/JWTVerifier.scala index 1c232645b7..b25ec55030 100644 --- a/otoroshi/app/models/JWTVerifier.scala +++ b/otoroshi/app/models/JWTVerifier.scala @@ -767,7 +767,8 @@ sealed trait JwtVerifier extends AsJson { "exp" -> Math.floor((System.currentTimeMillis() + 60) / 1000).toString, "sub" -> apikey.map(_.clientName).orElse(user.map(_.email)).getOrElse("anonymous"), "aud" -> "backend" - ) + ), + attrs = attrs ) .as[JsObject] val signedToken = sign(interpolatedToken, outputAlgorithm) @@ -899,7 +900,7 @@ sealed trait JwtVerifier extends AsJson { Some(desc), apikey, user, - context) + context, attrs) .as[JsObject] val newJsonToken: JsObject = JsObject( (tSettings.mappingSettings.map @@ -912,7 +913,7 @@ sealed trait JwtVerifier extends AsJson { Some(desc), apikey, user, - context)) + context, attrs)) .-(b._1) ) ++ evaluatedValues).fields .filterNot { diff --git a/otoroshi/app/models/config.scala b/otoroshi/app/models/config.scala index 922f96eb4c..1f8dc54132 100644 --- a/otoroshi/app/models/config.scala +++ b/otoroshi/app/models/config.scala @@ -1,8 +1,10 @@ package models +import akka.http.scaladsl.util.FastFuture import com.risksense.ipaddr.IpNetwork import env.Env import events._ +import otoroshi.plugins.geoloc.{IpStackGeolocationHelper, MaxMindGeolocationHelper} import play.api.Logger import play.api.libs.json._ import play.api.libs.ws.WSProxyServer @@ -168,6 +170,58 @@ object GlobalScripts { } } +object GeolocationSettings { + val format = new Format[GeolocationSettings] { + override def writes(o: GeolocationSettings): JsValue = o.json + override def reads(json: JsValue): JsResult[GeolocationSettings] = + Try { + JsSuccess( + (json \ "type").as[String] match { + case "none" => NoneGeolocationSettings + case "maxmind" => MaxmindGeolocationSettings( + enabled = (json \ "enabled").asOpt[Boolean].getOrElse(false), + path = (json \ "path").asOpt[String].filter(_.trim.nonEmpty).get + ) + case "ipstack" => IpStackGeolocationSettings( + enabled = (json \ "enabled").asOpt[Boolean].getOrElse(false), + apikey = (json \ "apikey").asOpt[String].filter(_.trim.nonEmpty).get, + timeout = (json \ "timeout").asOpt[Long].getOrElse(2000L) + ) + case _ => NoneGeolocationSettings + } + ) + } recover { + case e => JsError(e.getMessage) + } get + } +} + +sealed trait GeolocationSettings { + def enabled: Boolean + def find(ip: String)(implicit env: Env, ec: ExecutionContext): Future[Option[JsValue]] + def json: JsValue +} + +case object NoneGeolocationSettings extends GeolocationSettings { + def enabled: Boolean = false + def find(ip: String)(implicit env: Env, ec: ExecutionContext): Future[Option[JsValue]] = FastFuture.successful(None) + def json: JsValue = Json.obj("type" -> "none") +} + +case class MaxmindGeolocationSettings(enabled: Boolean, path: String) extends GeolocationSettings { + def json: JsValue = Json.obj("type" -> "maxmind", "path" -> path) + def find(ip: String)(implicit env: Env, ec: ExecutionContext): Future[Option[JsValue]] = { + MaxMindGeolocationHelper.find(ip, path) + } +} + +case class IpStackGeolocationSettings(enabled: Boolean, apikey: String, timeout: Long) extends GeolocationSettings { + def json: JsValue = Json.obj("type" -> "ipstack", "apikey" -> apikey, "timeout" -> timeout) + def find(ip: String)(implicit env: Env, ec: ExecutionContext): Future[Option[JsValue]] = { + IpStackGeolocationHelper.find(ip, apikey, timeout) + } +} + case class GlobalConfig( lines: Seq[String] = Seq("prod"), enableEmbeddedMetrics: Boolean = true, @@ -203,7 +257,8 @@ case class GlobalConfig( otoroshiId: String = IdGenerator.uuid, snowMonkeyConfig: SnowMonkeyConfig = SnowMonkeyConfig(), proxies: Proxies = Proxies(), - scripts: GlobalScripts = GlobalScripts() + scripts: GlobalScripts = GlobalScripts(), + geolocationSettings: GeolocationSettings = NoneGeolocationSettings ) { def save()(implicit ec: ExecutionContext, env: Env) = env.datastores.globalConfigDataStore.set(this) def delete()(implicit ec: ExecutionContext, env: Env) = env.datastores.globalConfigDataStore.delete(this) @@ -317,7 +372,8 @@ object GlobalConfig { "maxLogsSize" -> o.maxLogsSize, "otoroshiId" -> o.otoroshiId, "snowMonkeyConfig" -> o.snowMonkeyConfig.asJson, - "scripts" -> o.scripts.json + "scripts" -> o.scripts.json, + "geolocationSettings" -> o.geolocationSettings.json ) } override def reads(json: JsValue): JsResult[GlobalConfig] = @@ -428,7 +484,10 @@ object GlobalConfig { snowMonkeyConfig = (json \ "snowMonkeyConfig").asOpt(SnowMonkeyConfig._fmt).getOrElse(SnowMonkeyConfig()), scripts = GlobalScripts.format .reads((json \ "scripts").asOpt[JsValue].getOrElse(JsNull)) - .getOrElse(GlobalScripts()) + .getOrElse(GlobalScripts()), + geolocationSettings = GeolocationSettings.format + .reads((json \ "geolocationSettings").asOpt[JsValue].getOrElse(JsNull)) + .getOrElse(NoneGeolocationSettings) ) } map { case sd => JsSuccess(sd) diff --git a/otoroshi/app/models/descriptor.scala b/otoroshi/app/models/descriptor.scala index c2da7b5445..5d39458c35 100644 --- a/otoroshi/app/models/descriptor.scala +++ b/otoroshi/app/models/descriptor.scala @@ -792,8 +792,8 @@ object Canary { case class RedirectionSettings(enabled: Boolean = false, code: Int = 303, to: String = "https://www.otoroshi.io") { def toJson = RedirectionSettings.format.writes(this) def hasValidCode = RedirectionSettings.validRedirectionCodes.contains(code) - def formattedTo(request: RequestHeader, descriptor: ServiceDescriptor, ctx: Map[String, String]): String = - RedirectionExpressionLanguage(to, Some(request), Some(descriptor), None, None, ctx) + def formattedTo(request: RequestHeader, descriptor: ServiceDescriptor, ctx: Map[String, String], attrs: utils.TypedMap): String = + RedirectionExpressionLanguage(to, Some(request), Some(descriptor), None, None, ctx, attrs) } object RedirectionSettings { diff --git a/otoroshi/app/plugins/body.scala b/otoroshi/app/plugins/body.scala index 866d14be18..3699c21d12 100644 --- a/otoroshi/app/plugins/body.scala +++ b/otoroshi/app/plugins/body.scala @@ -56,11 +56,16 @@ case class RequestBodyEvent( method: String, url: String, headers: Map[String, String], - body: ByteString + body: ByteString, + from: String, + ua: String ) extends AnalyticEvent { override def `@type`: String = "RequestBodyEvent" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + def toJson(implicit _env: Env): JsValue = Json.obj( "@type" -> "RequestBodyEvent", "@id" -> `@id`, @@ -85,11 +90,16 @@ case class ResponseBodyEvent( url: String, headers: Map[String, String], status: Int, - body: ByteString + body: ByteString, + from: String, + ua: String ) extends AnalyticEvent { override def `@type`: String = "ResponseBodyEvent" + override def fromOrigin: Option[String] = Some(from) + override def fromUserAgent: Option[String] = Some(ua) + def toJson(implicit _env: Env): JsValue = Json.obj( "@type" -> "ResponseBodyEvent", "@id" -> `@id`, @@ -476,7 +486,9 @@ class BodyLogger extends RequestTransformer { method = ctx.rawRequest.method, url = ctx.rawRequest.url, headers = ctx.rawRequest.headers, - body = ref.get() + body = ref.get(), + from = ctx.request.headers.get("X-Forwarded-For").getOrElse(ctx.request.remoteAddress), + ua = ctx.request.headers.get("User-Agent").getOrElse("none") ) if (config.log) { event.log() @@ -518,7 +530,9 @@ class BodyLogger extends RequestTransformer { url = ctx.request.uri, headers = ctx.rawResponse.headers, status = ctx.rawResponse.status, - body = ref.get() + body = ref.get(), + from = ctx.request.headers.get("X-Forwarded-For").getOrElse(ctx.request.remoteAddress), + ua = ctx.request.headers.get("User-Agent").getOrElse("none") ) if (config.log) { event.log() diff --git a/otoroshi/app/storage/inmemory/InMemoryAlertDataStore.scala b/otoroshi/app/storage/inmemory/InMemoryAlertDataStore.scala index 186e2fc8a7..6c9c2051b4 100644 --- a/otoroshi/app/storage/inmemory/InMemoryAlertDataStore.scala +++ b/otoroshi/app/storage/inmemory/InMemoryAlertDataStore.scala @@ -3,7 +3,7 @@ package storage.inmemory import akka.util.ByteString import env.Env import events.{AlertDataStore, AlertEvent} -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import storage.RedisLike import scala.concurrent.{ExecutionContext, Future} @@ -17,10 +17,10 @@ class InMemoryAlertDataStore(redisCli: RedisLike) extends AlertDataStore { env: Env): Future[Seq[ByteString]] = redisCli.lrange(s"${env.storageRoot}:events:alerts", from, to) - override def push(event: AlertEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] = + override def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] = for { config <- env.datastores.globalConfigDataStore.singleton() - n <- redisCli.lpush(s"${env.storageRoot}:events:alerts", Json.stringify(event.toEnrichedJson)) + n <- redisCli.lpush(s"${env.storageRoot}:events:alerts", Json.stringify(event)) - <- redisCli.ltrim(s"${env.storageRoot}:events:alerts", 0, config.maxLogsSize) } yield n } diff --git a/otoroshi/app/storage/inmemory/InMemoryAuditDataStore.scala b/otoroshi/app/storage/inmemory/InMemoryAuditDataStore.scala index 89c3006698..3ad9b9d307 100644 --- a/otoroshi/app/storage/inmemory/InMemoryAuditDataStore.scala +++ b/otoroshi/app/storage/inmemory/InMemoryAuditDataStore.scala @@ -3,7 +3,7 @@ package storage.inmemory import akka.util.ByteString import env.Env import events.{AuditDataStore, AuditEvent} -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import storage.RedisLike import scala.concurrent.{ExecutionContext, Future} @@ -17,10 +17,10 @@ class InMemoryAuditDataStore(redisCli: RedisLike) extends AuditDataStore { env: Env): Future[Seq[ByteString]] = redisCli.lrange(s"${env.storageRoot}:events:audit", from, to) - override def push(event: AuditEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] = + override def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] = for { config <- env.datastores.globalConfigDataStore.singleton() - n <- redisCli.lpush(s"${env.storageRoot}:events:audit", Json.stringify(event.toEnrichedJson)) + n <- redisCli.lpush(s"${env.storageRoot}:events:audit", Json.stringify(event)) - <- redisCli.ltrim(s"${env.storageRoot}:events:audit", 0, config.maxLogsSize) } yield n } diff --git a/otoroshi/app/storage/inmemory/InMemoryHealthCheckDataStore.scala b/otoroshi/app/storage/inmemory/InMemoryHealthCheckDataStore.scala index 3a4e7a6bd9..44f571fe72 100644 --- a/otoroshi/app/storage/inmemory/InMemoryHealthCheckDataStore.scala +++ b/otoroshi/app/storage/inmemory/InMemoryHealthCheckDataStore.scala @@ -3,7 +3,7 @@ package storage.inmemory import env.Env import events.{HealthCheckDataStore, HealthCheckEvent} import models.ServiceDescriptor -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import storage.RedisLike import scala.concurrent.{ExecutionContext, Future} @@ -14,10 +14,10 @@ class InMemoryHealthCheckDataStore(redisCli: RedisLike, _env: Env) extends Healt def key(name: String) = s"${_env.storageRoot}:deschealthcheck:$name" - override def push(evt: HealthCheckEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] = + override def push(evt: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] = for { - n <- redisCli.lpush(key(evt.`@serviceId`), Json.stringify(evt.toEnrichedJson)) - _ <- redisCli.ltrim(key(evt.`@serviceId`), 0, collectionSize) + n <- redisCli.lpush(key((evt \ "@serviceId").as[String]), Json.stringify(evt)) + _ <- redisCli.ltrim(key((evt \ "@serviceId").as[String]), 0, collectionSize) } yield n override def findAll(serviceDescriptor: ServiceDescriptor)(implicit ec: ExecutionContext, diff --git a/otoroshi/app/storage/redis/RedisAlertDataStore.scala b/otoroshi/app/storage/redis/RedisAlertDataStore.scala index 9f786d63c8..68c7f18d43 100644 --- a/otoroshi/app/storage/redis/RedisAlertDataStore.scala +++ b/otoroshi/app/storage/redis/RedisAlertDataStore.scala @@ -3,7 +3,7 @@ package storage.redis import akka.util.ByteString import env.Env import events.{AlertDataStore, AlertEvent} -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import redis.RedisClientMasterSlaves import scala.concurrent.{ExecutionContext, Future} @@ -17,10 +17,10 @@ class RedisAlertDataStore(redisCli: RedisClientMasterSlaves) extends AlertDataSt env: Env): Future[Seq[ByteString]] = redisCli.lrange(s"${env.storageRoot}:events:alerts", from, to) - override def push(event: AlertEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] = + override def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] = for { config <- env.datastores.globalConfigDataStore.singleton() - n <- redisCli.lpush(s"${env.storageRoot}:events:alerts", Json.stringify(event.toJson)) + n <- redisCli.lpush(s"${env.storageRoot}:events:alerts", Json.stringify(event)) - <- redisCli.ltrim(s"${env.storageRoot}:events:alerts", 0, config.maxLogsSize) } yield n } diff --git a/otoroshi/app/storage/redis/RedisAuditDataStore.scala b/otoroshi/app/storage/redis/RedisAuditDataStore.scala index 74f3d3697b..0246b00a68 100644 --- a/otoroshi/app/storage/redis/RedisAuditDataStore.scala +++ b/otoroshi/app/storage/redis/RedisAuditDataStore.scala @@ -3,7 +3,7 @@ package storage.redis import akka.util.ByteString import env.Env import events.{AuditDataStore, AuditEvent} -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import redis.RedisClientMasterSlaves import scala.concurrent.{ExecutionContext, Future} @@ -17,10 +17,10 @@ class RedisAuditDataStore(redisCli: RedisClientMasterSlaves) extends AuditDataSt env: Env): Future[Seq[ByteString]] = redisCli.lrange(s"${env.storageRoot}:events:audit", from, to) - override def push(event: AuditEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] = + override def push(event: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] = for { config <- env.datastores.globalConfigDataStore.singleton() - n <- redisCli.lpush(s"${env.storageRoot}:events:audit", Json.stringify(event.toJson)) + n <- redisCli.lpush(s"${env.storageRoot}:events:audit", Json.stringify(event)) - <- redisCli.ltrim(s"${env.storageRoot}:events:audit", 0, config.maxLogsSize) } yield n } diff --git a/otoroshi/app/storage/redis/RedisHealthCheckDataStore.scala b/otoroshi/app/storage/redis/RedisHealthCheckDataStore.scala index cbdd746097..a9e53fd50a 100644 --- a/otoroshi/app/storage/redis/RedisHealthCheckDataStore.scala +++ b/otoroshi/app/storage/redis/RedisHealthCheckDataStore.scala @@ -1,11 +1,10 @@ package storage.redis import akka.http.scaladsl.util.FastFuture._ - import env.Env import events.{HealthCheckDataStore, HealthCheckEvent} import models.ServiceDescriptor -import play.api.libs.json.Json +import play.api.libs.json.{JsValue, Json} import redis.RedisClientMasterSlaves import scala.concurrent.{ExecutionContext, Future} @@ -16,10 +15,10 @@ class RedisHealthCheckDataStore(redisCli: RedisClientMasterSlaves, _env: Env) ex def key(name: String) = s"${_env.storageRoot}:deschealthcheck:$name" - override def push(evt: HealthCheckEvent)(implicit ec: ExecutionContext, env: Env): Future[Long] = + override def push(evt: JsValue)(implicit ec: ExecutionContext, env: Env): Future[Long] = for { - n <- redisCli.lpush(key(evt.`@serviceId`), Json.stringify(evt.toJson)) - _ <- redisCli.ltrim(key(evt.`@serviceId`), 0, collectionSize) + n <- redisCli.lpush(key((evt \ "@serviceId").as[String]), Json.stringify(evt)) + _ <- redisCli.ltrim(key((evt \ "@serviceId").as[String]), 0, collectionSize) } yield n override def findAll(serviceDescriptor: ServiceDescriptor)(implicit ec: ExecutionContext, diff --git a/otoroshi/app/utils/headers.scala b/otoroshi/app/utils/headers.scala index 82f8920031..b50e8e20ca 100644 --- a/otoroshi/app/utils/headers.scala +++ b/otoroshi/app/utils/headers.scala @@ -103,7 +103,8 @@ object HeadersHelper { stateToken: String, fromOtoroshi: Option[String], snowMonkeyContext: SnowMonkeyContext, - jwtInjection: JwtInjection + jwtInjection: JwtInjection, + attrs: utils.TypedMap )(implicit env: Env, ec: ExecutionContext): Seq[(String, String)] = { val stateRequestHeaderName = @@ -129,7 +130,8 @@ object HeadersHelper { snowMonkeyContext, jwtInjection, stateRequestHeaderName, - claimRequestHeaderName + claimRequestHeaderName, + attrs ) } else { @@ -140,13 +142,13 @@ object HeadersHelper { val missingOnlyHeaders: Seq[(String, String)] = descriptor.missingOnlyHeadersIn .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq val additionalHeaders: Seq[(String, String)] = descriptor.additionalHeaders .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq @@ -227,7 +229,8 @@ object HeadersHelper { overhead: Long, upstreamLatency: Long, canaryId: String, - remainingQuotas: RemainingQuotas + remainingQuotas: RemainingQuotas, + attrs: utils.TypedMap )(implicit env: Env, ec: ExecutionContext): Seq[(String, String)] = { val stateResponseHeaderName = descriptor.secComHeaders.stateResponseName @@ -248,7 +251,8 @@ object HeadersHelper { upstreamLatency, canaryId, remainingQuotas, - stateResponseHeaderName + stateResponseHeaderName, + attrs ) } else { @@ -259,20 +263,20 @@ object HeadersHelper { val missingOnlyHeadersOut = descriptor.missingOnlyHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq val additionalHeadersOut = descriptor.additionalHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq val corsHeaders = descriptor.cors .asHeaders(req) .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .map(v => (v._1, HeadersExpressionLanguage(v._2, Some(req), Some(descriptor), apiKey, paUsr, elCtx))) + .map(v => (v._1, HeadersExpressionLanguage(v._2, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs))) .filterNot(h => h._2 == "null") missingOnlyHeadersOut @@ -321,7 +325,8 @@ object HeadersHelper { overhead: Long, upstreamLatency: Long, canaryId: String, - remainingQuotas: RemainingQuotas + remainingQuotas: RemainingQuotas, + attrs: utils.TypedMap )(implicit env: Env, ec: ExecutionContext): Seq[(String, String)] = { val stateResponseHeaderName = descriptor.secComHeaders.stateResponseName @@ -342,7 +347,8 @@ object HeadersHelper { upstreamLatency, canaryId, remainingQuotas, - stateResponseHeaderName + stateResponseHeaderName, + attrs ) } else { @@ -352,20 +358,20 @@ object HeadersHelper { val missingOnlyHeadersOut = descriptor.missingOnlyHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq val additionalHeadersOut = descriptor.additionalHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq val corsHeaders = descriptor.cors .asHeaders(req) .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .map(v => (v._1, HeadersExpressionLanguage(v._2, Some(req), Some(descriptor), apiKey, paUsr, elCtx))) + .map(v => (v._1, HeadersExpressionLanguage(v._2, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs))) .filterNot(h => h._2 == "null") missingOnlyHeadersOut @@ -421,12 +427,13 @@ object HeadersHelper { snowMonkeyContext: SnowMonkeyContext, jwtInjection: JwtInjection, stateRequestHeaderName: String, - claimRequestHeaderName: String + claimRequestHeaderName: String, + attrs: utils.TypedMap )(implicit env: Env, ec: ExecutionContext): Seq[(String, String)] = { val headersIn: Seq[(String, String)] = { (descriptor.missingOnlyHeadersIn .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") ++ req.headers.toMap.toSeq .flatMap(c => c._2.map(v => (c._1, v))) //.map(tuple => (tuple._1, tuple._2.mkString(","))) //.toSimpleMap @@ -474,7 +481,7 @@ object HeadersHelper { .getOrElse(Map.empty[String, String]) ++ descriptor.additionalHeaders .filter(t => t._1.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") ++ fromOtoroshi .map(v => Map(env.Headers.OtoroshiGatewayParentRequest -> fromOtoroshi.get)) .getOrElse(Map.empty[String, String]) ++ jwtInjection.additionalHeaders).toSeq @@ -501,7 +508,8 @@ object HeadersHelper { upstreamLatency: Long, canaryId: String, remainingQuotas: RemainingQuotas, - stateResponseHeaderName: String + stateResponseHeaderName: String, + attrs: utils.TypedMap )(implicit env: Env, ec: ExecutionContext): Seq[(String, String)] = { val _headersForOut: Seq[(String, String)] = resp.headers.toSeq.flatMap( @@ -510,7 +518,7 @@ object HeadersHelper { val _headersOut: Seq[(String, String)] = { descriptor.missingOnlyHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq ++ _headersForOut @@ -541,7 +549,7 @@ object HeadersHelper { }) ++ descriptor.cors .asHeaders(req) ++ descriptor.additionalHeadersOut .mapValues( - v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx) + v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs) ) .filterNot(h => h._2 == "null") .toSeq @@ -564,12 +572,13 @@ object HeadersHelper { upstreamLatency: Long, canaryId: String, remainingQuotas: RemainingQuotas, - stateResponseHeaderName: String + stateResponseHeaderName: String, + attrs: utils.TypedMap )(implicit env: Env, ec: ExecutionContext): Seq[(String, String)] = { val _headersOut: Seq[(String, String)] = { descriptor.missingOnlyHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) - .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx)) + .mapValues(v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs)) .filterNot(h => h._2 == "null") .toSeq ++ badResult.header.headers.toSeq @@ -603,7 +612,7 @@ object HeadersHelper { }) ++ descriptor.cors.asHeaders(req) ++ descriptor.additionalHeadersOut .filter(t => t._1.trim.nonEmpty && t._2.trim.nonEmpty) .mapValues( - v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx) + v => HeadersExpressionLanguage.apply(v, Some(req), Some(descriptor), apiKey, paUsr, elCtx, attrs) ) .filterNot(h => h._2 == "null") .toSeq diff --git a/otoroshi/test/functional/AnalyticsSpec.scala b/otoroshi/test/functional/AnalyticsSpec.scala index 61371c129d..d9bb33079c 100644 --- a/otoroshi/test/functional/AnalyticsSpec.scala +++ b/otoroshi/test/functional/AnalyticsSpec.scala @@ -354,7 +354,7 @@ class AnalyticsSpec(name: String, configurationSpec: => Configuration) def setUpEvent(seq: AnalyticEvent*): Unit = { implicit val ec: ExecutionContext = otoroshiComponents.executionContext implicit val env: Env = otoroshiComponents.env - analytics.publish(seq).futureValue + analytics.publish(seq.map(_.toJson)).futureValue } private def getStatus(i: Int): Int = {