From f04a2733711358794315c55f8e9696a1015556c1 Mon Sep 17 00:00:00 2001 From: Camelia Dumitru Date: Wed, 22 Jan 2025 13:21:08 +0000 Subject: [PATCH] RateLimitig with Redis Implementation --- .../rate_limit/PapiRateLimitRedisClient.java | 95 ++++++++++++++ .../core/utils/cache/redis/RedisClient.java | 97 +++++++++----- .../src/main/resources/orcid-core-context.xml | 12 ++ .../PublicApiDailyRateLimitEntity.java | 4 +- .../orcid/api/filters/ApiRateLimitFilter.java | 119 +++++++++--------- .../api/filters/ApiRateLimitFilterTest.java | 104 ++++++++------- .../report/PapiDailyLimitReport.java | 16 ++- .../resources/properties/test-core.properties | 10 +- properties/development.properties | 12 +- 9 files changed, 326 insertions(+), 143 deletions(-) create mode 100644 orcid-core/src/main/java/org/orcid/core/api/rate_limit/PapiRateLimitRedisClient.java diff --git a/orcid-core/src/main/java/org/orcid/core/api/rate_limit/PapiRateLimitRedisClient.java b/orcid-core/src/main/java/org/orcid/core/api/rate_limit/PapiRateLimitRedisClient.java new file mode 100644 index 00000000000..6879b855b57 --- /dev/null +++ b/orcid-core/src/main/java/org/orcid/core/api/rate_limit/PapiRateLimitRedisClient.java @@ -0,0 +1,95 @@ +package org.orcid.core.api.rate_limit; + +import java.time.LocalDate; +import java.util.Date; +import java.util.HashMap; + +import javax.annotation.Resource; + +import org.apache.commons.lang.StringUtils; +import org.orcid.core.utils.cache.redis.RedisClient; +import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; +import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; + +@Component +public class PapiRateLimitRedisClient { + private static final Logger LOG = LoggerFactory.getLogger(PapiRateLimitRedisClient.class); + + @Resource(name = "redisClientPapi") + private RedisClient redisClient; + + @Value("${org.orcid.papi.rate.limit.redisCacheExpiryInSec:172800}") + private int CASH_EXPIRY_IN_SECONDS; // caching for 2 days to have time to + // synch with DB + + @Autowired + private PublicApiDailyRateLimitDao papiRateLimitingDao; + + public static final String KEY_REQUEST_COUNT = "reqCount"; + public static final String KEY_REQUEST_DATE = "reqDate"; + public static final String KEY_LAST_MODIFIED = "lastModified"; + public static final String KEY_DATE_CREATED = "dateCreated"; + public static final String KEY_IS_ANONYMOUS = "isAnonymous"; + public static final String KEY_REQUEST_CLIENT = "reqClient"; + + public static final String KEY_DELIMITATOR = "||"; + + public String getTodayKeyByClient(String client) { + return getRequestDateKeyByClient(client, LocalDate.now()); + } + + public String getRequestDateKeyByClient(String client, LocalDate requestDate) { + return client + KEY_DELIMITATOR + requestDate.toString() ; + } + + public JSONObject getDailyLimitsForClient(String client, LocalDate requestDate) { + + String metaData = redisClient.get(getRequestDateKeyByClient(client, requestDate)); + try { + return StringUtils.isNotBlank(metaData) ? new JSONObject(metaData) : null; + } catch (Exception ex) { + return null; + } + } + + public JSONObject getTodayDailyLimitsForClient(String client) { + return getDailyLimitsForClient(client, LocalDate.now()); + } + + public void setTodayLimitsForClient(String client, JSONObject metaData) { + String limitKey = getTodayKeyByClient(client); + redisClient.set(limitKey, metaData.toString(), CASH_EXPIRY_IN_SECONDS); + } + + public void saveRedisPapiLimitDateToDB(LocalDate requestDate) throws JSONException { + // returns all the keys for requestDate + HashMap allValuesForKey = redisClient.getAllValuesForKeyPattern("*" + requestDate.toString()); + for (String key : allValuesForKey.keySet()) { + papiRateLimitingDao.persist(redisObjJsonToEntity(allValuesForKey.get(key))); + redisClient.remove(key); + } + } + + private PublicApiDailyRateLimitEntity redisObjJsonToEntity(JSONObject redisObj) throws JSONException { + PublicApiDailyRateLimitEntity rateLimitEntity = new PublicApiDailyRateLimitEntity(); + if (!redisObj.getBoolean(KEY_IS_ANONYMOUS)) { + rateLimitEntity.setClientId(redisObj.getString(KEY_REQUEST_CLIENT)); + } else { + rateLimitEntity.setIpAddress(redisObj.getString(KEY_REQUEST_CLIENT)); + } + rateLimitEntity.setRequestCount(redisObj.getLong(KEY_REQUEST_COUNT)); + rateLimitEntity.setRequestDate(LocalDate.parse(redisObj.getString(KEY_REQUEST_DATE))); + rateLimitEntity.setDateCreated(new Date(redisObj.getInt(KEY_DATE_CREATED))); + rateLimitEntity.setLastModified(new Date(redisObj.getInt(KEY_LAST_MODIFIED))); + papiRateLimitingDao.persist(rateLimitEntity); + return rateLimitEntity; + } + +} diff --git a/orcid-core/src/main/java/org/orcid/core/utils/cache/redis/RedisClient.java b/orcid-core/src/main/java/org/orcid/core/utils/cache/redis/RedisClient.java index 4d62057301b..4980e035f46 100644 --- a/orcid-core/src/main/java/org/orcid/core/utils/cache/redis/RedisClient.java +++ b/orcid-core/src/main/java/org/orcid/core/utils/cache/redis/RedisClient.java @@ -4,10 +4,14 @@ import java.net.InetAddress; import java.net.SocketException; import java.net.UnknownHostException; +import java.util.HashMap; +import java.util.List; import javax.annotation.PostConstruct; import javax.annotation.Resource; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.orcid.utils.alerting.SlackManager; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -17,15 +21,18 @@ import redis.clients.jedis.Jedis; import redis.clients.jedis.JedisClientConfig; import redis.clients.jedis.JedisPool; +import redis.clients.jedis.params.ScanParams; import redis.clients.jedis.params.SetParams; +import redis.clients.jedis.resps.ScanResult; public class RedisClient { private static final Logger LOG = LoggerFactory.getLogger(RedisClient.class); - + private static final int DEFAULT_CACHE_EXPIRY = 60; private static final int DEFAULT_TIMEOUT = 10000; - + public static final int MACH_KEY_BATCH_SIZE = 1000; + private final String redisHost; private final int redisPort; private final String redisPassword; @@ -33,19 +40,19 @@ public class RedisClient { private final int clientTimeoutInMillis; private JedisPool pool; private SetParams defaultSetParams; - + @Resource private SlackManager slackManager; - - // Assume the connection to Redis is disabled by default + + // Assume the connection to Redis is disabled by default private boolean enabled = false; - + public RedisClient(String redisHost, int redisPort, String password) { this.redisHost = redisHost; this.redisPort = redisPort; this.redisPassword = password; this.cacheExpiryInSecs = DEFAULT_CACHE_EXPIRY; - this.clientTimeoutInMillis = DEFAULT_TIMEOUT; + this.clientTimeoutInMillis = DEFAULT_TIMEOUT; } public RedisClient(String redisHost, int redisPort, String password, int cacheExpiryInSecs) { @@ -53,7 +60,7 @@ public RedisClient(String redisHost, int redisPort, String password, int cacheEx this.redisPort = redisPort; this.redisPassword = password; this.cacheExpiryInSecs = cacheExpiryInSecs; - this.clientTimeoutInMillis = DEFAULT_TIMEOUT; + this.clientTimeoutInMillis = DEFAULT_TIMEOUT; } public RedisClient(String redisHost, int redisPort, String password, int cacheExpiryInSecs, int clientTimeoutInMillis) { @@ -68,32 +75,33 @@ public RedisClient(String redisHost, int redisPort, String password, int cacheEx private void init() { try { JedisClientConfig config = DefaultJedisClientConfig.builder().connectionTimeoutMillis(this.clientTimeoutInMillis).timeoutMillis(this.clientTimeoutInMillis) - .socketTimeoutMillis(this.clientTimeoutInMillis).password(this.redisPassword).ssl(true).build(); - pool = new JedisPool(new HostAndPort(this.redisHost, this.redisPort), config); - defaultSetParams = new SetParams().ex(this.cacheExpiryInSecs); + .socketTimeoutMillis(this.clientTimeoutInMillis).password(this.redisPassword).ssl(true).build(); + pool = new JedisPool(new HostAndPort(this.redisHost, this.redisPort), config); + defaultSetParams = new SetParams().ex(this.cacheExpiryInSecs); // Pool test - try(Jedis jedis = pool.getResource()) { - if(jedis.isConnected()) { + try (Jedis jedis = pool.getResource()) { + if (jedis.isConnected()) { LOG.info("Connected to the Redis cache, elements will be cached for " + this.cacheExpiryInSecs + " seconds"); - // As it was possible to make the connection, enable the client + // As it was possible to make the connection, enable the + // client enabled = true; } } - } catch(Exception e) { + } catch (Exception e) { LOG.error("Exception initializing Redis client", e); try { - // Lets try to get the host name - InetAddress id = InetAddress.getLocalHost(); + // Lets try to get the host name + InetAddress id = InetAddress.getLocalHost(); slackManager.sendSystemAlert("Unable to start Redis client on " + id.getHostName()); - } catch(UnknownHostException uhe) { + } catch (UnknownHostException uhe) { // Lets try to get the IP address - try(final DatagramSocket socket = new DatagramSocket()){ + try (final DatagramSocket socket = new DatagramSocket()) { socket.connect(InetAddress.getByName("8.8.8.8"), 10002); String ip = socket.getLocalAddress().getHostAddress(); slackManager.sendSystemAlert("Unable to start Redis client on IP " + ip); - } catch(SocketException | UnknownHostException se) { - slackManager.sendSystemAlert("Unable to start Redis client - Couldn't identify the machine"); - } + } catch (SocketException | UnknownHostException se) { + slackManager.sendSystemAlert("Unable to start Redis client - Couldn't identify the machine"); + } } } } @@ -101,14 +109,14 @@ private void init() { public boolean set(String key, String value) { return set(key, value, defaultSetParams); } - + public boolean set(String key, String value, int cacheExpiryInSecs) { SetParams params = new SetParams().ex(cacheExpiryInSecs); return set(key, value, params); } private boolean set(String key, String value, SetParams params) { - if(enabled && pool != null) { + if (enabled && pool != null) { try (Jedis jedis = pool.getResource()) { LOG.debug("Setting Key: {}", key); String result = jedis.set(key, value, params); @@ -117,17 +125,17 @@ private boolean set(String key, String value, SetParams params) { } return false; } - + public String get(String key) { - if(enabled && pool != null) { + if (enabled && pool != null) { try (Jedis jedis = pool.getResource()) { - LOG.debug("Reading Key: {}" , key); + LOG.debug("Reading Key: {}", key); return jedis.get(key); } - } + } return null; } - + public boolean remove(String key) { if (enabled && pool != null) { try (Jedis jedis = pool.getResource()) { @@ -141,4 +149,35 @@ public boolean remove(String key) { } return true; } + + /** + * Retrieve the mapped key, value for all the keys that match the matchKey + * parameter. + * + * @param machKey + * the key pattern for which the mapped values are returned + * @return + * @throws JSONException + */ + public HashMap getAllValuesForKeyPattern(String matchKey) throws JSONException { + HashMap mappedValuesForKey = new HashMap(); + // Connect to Redis + try (Jedis jedis = pool.getResource()) { + String cursor = "0"; + ScanParams scanParams = new ScanParams().match(matchKey).count(MACH_KEY_BATCH_SIZE); + do { + // Use SCAN to fetch matching keys in batches + ScanResult scanResult = jedis.scan(cursor, scanParams); + cursor = scanResult.getCursor(); + List keys = scanResult.getResult(); + + // Print each key and its corresponding value + for (String key : keys) { + mappedValuesForKey.put(key, new JSONObject(jedis.get(key))); + } + } while (!"0".equals(cursor)); // SCAN ends when cursor returns "0" + } + + return mappedValuesForKey; + } } diff --git a/orcid-core/src/main/resources/orcid-core-context.xml b/orcid-core/src/main/resources/orcid-core-context.xml index ebccc63371a..1b1725537be 100644 --- a/orcid-core/src/main/resources/orcid-core-context.xml +++ b/orcid-core/src/main/resources/orcid-core-context.xml @@ -1228,6 +1228,18 @@ + + + + + + + + + + + + diff --git a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java index bc8a9d42ec8..0a3758f5653 100644 --- a/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java +++ b/orcid-persistence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java @@ -93,7 +93,7 @@ public Date getDateCreated() { return dateCreated; } - void setDateCreated(Date date) { + public void setDateCreated(Date date) { this.dateCreated = date; } @@ -101,7 +101,7 @@ public Date getLastModified() { return lastModified; } - void setLastModified(Date lastModified) { + public void setLastModified(Date lastModified) { this.lastModified = lastModified; } diff --git a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java index 8767eb0d162..16cf9e5ff88 100644 --- a/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java +++ b/orcid-pub-web/src/main/java/org/orcid/api/filters/ApiRateLimitFilter.java @@ -14,15 +14,16 @@ import org.apache.commons.lang.LocaleUtils; import org.apache.commons.lang3.StringUtils; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.orcid.core.api.rate_limit.PapiRateLimitRedisClient; import org.orcid.core.manager.ClientDetailsEntityCacheManager; import org.orcid.core.manager.TemplateManager; import org.orcid.core.manager.impl.OrcidUrlManager; import org.orcid.core.manager.v3.EmailManager; import org.orcid.core.manager.v3.RecordNameManager; import org.orcid.core.oauth.service.OrcidTokenStore; -import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; import org.orcid.persistence.jpa.entities.ClientDetailsEntity; -import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; import org.orcid.utils.email.MailGunManager; import org.orcid.utils.panoply.PanoplyPapiDailyRateExceededItem; import org.orcid.utils.panoply.PanoplyRedshiftClient; @@ -42,9 +43,6 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { private static final Logger LOG = LoggerFactory.getLogger(ApiRateLimitFilter.class); - @Autowired - private PublicApiDailyRateLimitDao papiRateLimitingDao; - @Autowired private ClientDetailsEntityCacheManager clientDetailsEntityCacheManager; @@ -66,6 +64,9 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Resource private PanoplyRedshiftClient panoplyClient; + @Resource + private PapiRateLimitRedisClient papiRedisClient; + @Autowired private OrcidTokenStore orcidTokenStore; @@ -84,12 +85,13 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { @Value("${org.orcid.persistence.panoply.papiExceededRate.production:false}") private boolean enablePanoplyPapiExceededRateInProduction; + // :192.168.65.1 127.0.0.1 @Value("${org.orcid.papi.rate.limit.ip.whiteSpaceSeparatedWhiteList:192.168.65.1 127.0.0.1}") private String papiWhiteSpaceSeparatedWhiteList; @Value("${org.orcid.papi.rate.limit.clientId.whiteSpaceSeparatedWhiteList}") private String papiClientIdWhiteSpaceSeparatedWhiteList; - + @Value("${org.orcid.papi.rate.limit.referrer.whiteSpaceSeparatedWhiteList}") private String papiReferrerWhiteSpaceSeparatedWhiteList; @@ -113,14 +115,16 @@ public class ApiRateLimitFilter extends OncePerRequestFilter { public void afterPropertiesSet() throws ServletException { super.afterPropertiesSet(); papiIpWhiteList = StringUtils.isNotBlank(papiWhiteSpaceSeparatedWhiteList) ? Arrays.asList(papiWhiteSpaceSeparatedWhiteList.split("\\s")) : null; - papiClientIdWhiteList = StringUtils.isNotBlank(papiClientIdWhiteSpaceSeparatedWhiteList) ? Arrays.asList(papiClientIdWhiteSpaceSeparatedWhiteList.split("\\s")) : null; - papiReferrerWhiteList = StringUtils.isNotBlank(papiReferrerWhiteSpaceSeparatedWhiteList) ? Arrays.asList(papiReferrerWhiteSpaceSeparatedWhiteList.split("\\s")) : null; + papiClientIdWhiteList = StringUtils.isNotBlank(papiClientIdWhiteSpaceSeparatedWhiteList) ? Arrays.asList(papiClientIdWhiteSpaceSeparatedWhiteList.split("\\s")) + : null; + papiReferrerWhiteList = StringUtils.isNotBlank(papiReferrerWhiteSpaceSeparatedWhiteList) ? Arrays.asList(papiReferrerWhiteSpaceSeparatedWhiteList.split("\\s")) + : null; } @Override public void doFilterInternal(HttpServletRequest httpServletRequest, HttpServletResponse httpServletResponse, FilterChain filterChain) throws ServletException, IOException { - LOG.trace("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); + LOG.warn("ApiRateLimitFilter starts, rate limit is : " + enableRateLimiting); if (enableRateLimiting && !isReferrerWhiteListed(httpServletRequest.getHeader(HttpHeaders.REFERER))) { String tokenValue = null; @@ -160,61 +164,51 @@ public void doFilterInternal(HttpServletRequest httpServletRequest, HttpServletR filterChain.doFilter(httpServletRequest, httpServletResponse); } - private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpServletResponse httpServletResponse) throws IOException { - PublicApiDailyRateLimitEntity rateLimitEntity = papiRateLimitingDao.findByIpAddressAndRequestDate(ipAddress, today); - if (rateLimitEntity != null) { - // update the request count only when limit not exceeded ? - rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); - papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, false); - if (rateLimitEntity.getRequestCount() == knownRequestLimit && enablePanoplyPapiExceededRateInProduction) { - PanoplyPapiDailyRateExceededItem item = new PanoplyPapiDailyRateExceededItem(); - item.setIpAddress(ipAddress); - item.setRequestDate(rateLimitEntity.getRequestDate()); - setPapiRateExceededItemInPanoply(item); - } - if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { - if (rateLimitEntity.getRequestCount() >= anonymousRequestLimit) { - httpServletResponse.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); - if (!httpServletResponse.isCommitted()) { - try (PrintWriter writer = httpServletResponse.getWriter()) { - writer.write(TOO_MANY_REQUESTS_MSG); - writer.flush(); - } - return; - } + private void rateLimitAnonymousRequest(String ipAddress, LocalDate today, HttpServletResponse httpServletResponse) throws IOException, JSONException { + JSONObject dailyLimitsObj = papiRedisClient.getTodayDailyLimitsForClient(ipAddress); + long limitValue = 0l; + if (dailyLimitsObj != null) { + limitValue = dailyLimitsObj.getLong(PapiRateLimitRedisClient.KEY_REQUEST_COUNT); + } else { + dailyLimitsObj = new JSONObject(); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_DATE_CREATED, System.currentTimeMillis()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_IS_ANONYMOUS, true); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_DATE, today.toString()); + } + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_CLIENT, ipAddress); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_COUNT, limitValue + 1); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_LAST_MODIFIED, System.currentTimeMillis()); + papiRedisClient.setTodayLimitsForClient(ipAddress, dailyLimitsObj); + if (Features.ENABLE_PAPI_RATE_LIMITING.isActive() && (limitValue + 1) >= anonymousRequestLimit) { + httpServletResponse.setStatus(HttpStatus.TOO_MANY_REQUESTS.value()); + if (!httpServletResponse.isCommitted()) { + try (PrintWriter writer = httpServletResponse.getWriter()) { + writer.write(TOO_MANY_REQUESTS_MSG); + writer.flush(); } + return; } - } else { - // create - rateLimitEntity = new PublicApiDailyRateLimitEntity(); - rateLimitEntity.setIpAddress(ipAddress); - rateLimitEntity.setRequestCount(1L); - rateLimitEntity.setRequestDate(today); - papiRateLimitingDao.persist(rateLimitEntity); - } - return; } - private void rateLimitClientRequest(String clientId, LocalDate today) { - PublicApiDailyRateLimitEntity rateLimitEntity = papiRateLimitingDao.findByClientIdAndRequestDate(clientId, today); - if (rateLimitEntity != null) { - if (Features.ENABLE_PAPI_RATE_LIMITING.isActive()) { - // email the client first time the limit is reached - if (rateLimitEntity.getRequestCount() == knownRequestLimit) { - sendEmail(clientId, rateLimitEntity.getRequestDate()); - } - } - // update the request count - rateLimitEntity.setRequestCount(rateLimitEntity.getRequestCount() + 1); - papiRateLimitingDao.updatePublicApiDailyRateLimit(rateLimitEntity, true); + private void rateLimitClientRequest(String clientId, LocalDate today) throws JSONException { + JSONObject dailyLimitsObj = papiRedisClient.getTodayDailyLimitsForClient(clientId); + long limitValue = 0l; + if (dailyLimitsObj != null) { + limitValue = dailyLimitsObj.getLong(PapiRateLimitRedisClient.KEY_REQUEST_COUNT); } else { - // create - rateLimitEntity = new PublicApiDailyRateLimitEntity(); - rateLimitEntity.setClientId(clientId); - rateLimitEntity.setRequestCount(1L); - rateLimitEntity.setRequestDate(today); - papiRateLimitingDao.persist(rateLimitEntity); + dailyLimitsObj = new JSONObject(); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_DATE_CREATED, System.currentTimeMillis()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_IS_ANONYMOUS, true); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_DATE, today.toString()); + } + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_CLIENT, clientId); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_COUNT, limitValue + 1); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_LAST_MODIFIED, System.currentTimeMillis()); + + papiRedisClient.setTodayLimitsForClient(clientId, dailyLimitsObj); + if (Features.ENABLE_PAPI_RATE_LIMITING.isActive() && (limitValue == knownRequestLimit)) { + sendEmail(clientId, LocalDate.now()); } } @@ -280,7 +274,8 @@ private void setPapiRateExceededItemInPanoply(PanoplyPapiDailyRateExceededItem i }); } - // gets actual client IP address, using the headers that the proxy server adds + // gets actual client IP address, using the headers that the proxy server + // adds private String getClientIpAddress(HttpServletRequest request) { String ipAddress = request.getHeader("X-FORWARDED-FOR"); if (ipAddress == null || ipAddress.isEmpty() || "unknown".equalsIgnoreCase(ipAddress)) { @@ -300,13 +295,13 @@ private boolean isWhiteListed(String ipAddress) { } private boolean isClientIdWhiteListed(String clientId) { - return (papiClientIdWhiteList != null) ? papiClientIdWhiteList.contains(clientId) :false; + return (papiClientIdWhiteList != null) ? papiClientIdWhiteList.contains(clientId) : false; } - + private boolean isReferrerWhiteListed(String referrer) { if (referrer == null) return false; else - return (papiReferrerWhiteList != null) ? papiReferrerWhiteList.contains(referrer) :false; + return (papiReferrerWhiteList != null) ? papiReferrerWhiteList.contains(referrer) : false; } } diff --git a/orcid-pub-web/src/test/java/org/orcid/api/filters/ApiRateLimitFilterTest.java b/orcid-pub-web/src/test/java/org/orcid/api/filters/ApiRateLimitFilterTest.java index 064e7590435..427a4dcaa23 100644 --- a/orcid-pub-web/src/test/java/org/orcid/api/filters/ApiRateLimitFilterTest.java +++ b/orcid-pub-web/src/test/java/org/orcid/api/filters/ApiRateLimitFilterTest.java @@ -1,9 +1,12 @@ package org.orcid.api.filters; +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; import org.junit.Test; import org.junit.runner.RunWith; import org.mockito.Mock; import org.mockito.MockitoAnnotations; +import org.orcid.core.api.rate_limit.PapiRateLimitRedisClient; import org.orcid.core.oauth.service.OrcidTokenStore; import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity; @@ -16,6 +19,7 @@ import javax.servlet.FilterChain; import javax.servlet.ServletException; import java.io.IOException; +import java.time.LocalDate; import static org.junit.Assert.assertEquals; import static org.mockito.ArgumentMatchers.eq; @@ -35,7 +39,7 @@ public class ApiRateLimitFilterTest { private OrcidTokenStore orcidTokenStoreMock; @Mock - private PublicApiDailyRateLimitDao papiRateLimitingDaoMock; + private PapiRateLimitRedisClient papiRateLimitRedisMock; MockHttpServletRequest httpServletRequestMock = new MockHttpServletRequest(); @@ -46,13 +50,15 @@ public void doFilterInternal_rateLimitingDisabledTest() throws ServletException, MockitoAnnotations.initMocks(this); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", false); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); verify(filterChainMock, times(1)).doFilter(eq(httpServletRequestMock), eq(httpServletResponseMock)); verify(orcidTokenStoreMock, never()).readClientId(anyString()); - verify(papiRateLimitingDaoMock, never()).findByIpAddressAndRequestDate(anyString(), any()); - verify(papiRateLimitingDaoMock, never()).persist(any()); + + verify(papiRateLimitRedisMock, never()).getDailyLimitsForClient(anyString(), any()); + verify(papiRateLimitRedisMock, never()).setTodayLimitsForClient(anyString(), any()); } @Test @@ -62,16 +68,15 @@ public void doFilterInternal_annonymousRequest_newEntry_X_FORWARDED_FOR_header_T TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByIpAddressAndRequestDate(eq(ip), any())).thenReturn(null); + when(papiRateLimitRedisMock.getTodayDailyLimitsForClient(eq(ip))).thenReturn(null); httpServletRequestMock.addHeader("X-FORWARDED-FOR", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); verify(orcidTokenStoreMock, never()).readClientId(anyString()); - verify(papiRateLimitingDaoMock, never()).updatePublicApiDailyRateLimit(any(PublicApiDailyRateLimitEntity.class), anyBoolean()); - verify(papiRateLimitingDaoMock, times(1)).persist(any(PublicApiDailyRateLimitEntity.class)); + verify(papiRateLimitRedisMock, times(1)).setTodayLimitsForClient(anyString(), any(JSONObject.class)); } @Test @@ -81,16 +86,17 @@ public void doFilterInternal_annonymousRequest_newEntry_X_REAL_IP_header_Test() TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByIpAddressAndRequestDate(eq(ip), any())).thenReturn(null); + when(papiRateLimitRedisMock.getTodayDailyLimitsForClient(eq(ip))).thenReturn(null); httpServletRequestMock.addHeader("X-REAL-IP", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); verify(orcidTokenStoreMock, never()).readClientId(anyString()); - verify(papiRateLimitingDaoMock, never()).updatePublicApiDailyRateLimit(any(PublicApiDailyRateLimitEntity.class), anyBoolean()); - verify(papiRateLimitingDaoMock, times(1)).persist(any(PublicApiDailyRateLimitEntity.class)); + + verify(papiRateLimitRedisMock, never()).getDailyLimitsForClient(anyString(), any()); + verify(papiRateLimitRedisMock, times(1)).setTodayLimitsForClient(anyString(), any(JSONObject.class)); } @Test @@ -100,39 +106,41 @@ public void doFilterInternal_annonymousRequest_newEntry_whitelisted_IP_Test() th TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByIpAddressAndRequestDate(eq(ip), any())).thenReturn(null); + when(papiRateLimitRedisMock.getTodayDailyLimitsForClient(eq(ip))).thenReturn(null); httpServletRequestMock.addHeader("X-REAL-IP", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); verify(orcidTokenStoreMock, never()).readClientId(anyString()); - verify(papiRateLimitingDaoMock, never()).updatePublicApiDailyRateLimit(any(PublicApiDailyRateLimitEntity.class), anyBoolean()); - verify(papiRateLimitingDaoMock, never()).persist(any(PublicApiDailyRateLimitEntity.class)); + verify(papiRateLimitRedisMock, never()).setTodayLimitsForClient(eq(ip), any()); } @Test - public void doFilterInternal_annonymousRequest_existingEntryTest() throws ServletException, IOException { + public void doFilterInternal_annonymousRequest_existingEntryTest() throws ServletException, IOException, JSONException { MockitoAnnotations.initMocks(this); String ip = "127.0.0.2"; - PublicApiDailyRateLimitEntity e = new PublicApiDailyRateLimitEntity(); - e.setId(1000L); - e.setIpAddress(ip); - e.setRequestCount(100L); + JSONObject dailyLimitsObj = new JSONObject(); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_DATE_CREATED, System.currentTimeMillis()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_IS_ANONYMOUS, true); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_DATE, LocalDate.now().toString()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_CLIENT, ip); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_COUNT, 1); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_LAST_MODIFIED, System.currentTimeMillis()); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByIpAddressAndRequestDate(eq(ip), any())).thenReturn(e); + when(papiRateLimitRedisMock.getTodayDailyLimitsForClient(eq(ip))).thenReturn(dailyLimitsObj); httpServletRequestMock.addHeader("X-REAL-IP", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); verify(orcidTokenStoreMock, never()).readClientId(anyString()); - verify(papiRateLimitingDaoMock, times(1)).updatePublicApiDailyRateLimit(any(PublicApiDailyRateLimitEntity.class), eq(false)); - verify(papiRateLimitingDaoMock, never()).persist(any(PublicApiDailyRateLimitEntity.class)); + verify(papiRateLimitRedisMock, times(1)).setTodayLimitsForClient(anyString(), any(JSONObject.class)); + } @Test @@ -145,64 +153,70 @@ public void doFilterInternal_clientRequest_newEntryTest() throws ServletExceptio when(orcidTokenStoreMock.readClientId(eq("TEST_TOKEN"))).thenReturn(clientId); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByClientIdAndRequestDate(eq(ip), any())).thenReturn(null); + when(papiRateLimitRedisMock.getTodayDailyLimitsForClient(eq(ip))).thenReturn(null); httpServletRequestMock.addHeader("X-REAL-IP", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); - verify(papiRateLimitingDaoMock, never()).updatePublicApiDailyRateLimit(any(PublicApiDailyRateLimitEntity.class), anyBoolean()); - verify(papiRateLimitingDaoMock, times(1)).persist(any(PublicApiDailyRateLimitEntity.class)); + verify(papiRateLimitRedisMock, times(1)).setTodayLimitsForClient(anyString(), any(JSONObject.class)); } @Test - public void doFilterInternal_clientRequest_existingEntryTest() throws ServletException, IOException { + public void doFilterInternal_clientRequest_existingEntryTest() throws ServletException, IOException, JSONException { MockitoAnnotations.initMocks(this); String ip = "127.0.0.2"; String clientId = "clientId1"; - PublicApiDailyRateLimitEntity e = new PublicApiDailyRateLimitEntity(); - e.setId(1000L); - e.setIpAddress(ip); - e.setRequestCount(100L); + JSONObject dailyLimitsObj = new JSONObject(); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_DATE_CREATED, System.currentTimeMillis()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_IS_ANONYMOUS, true); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_DATE, LocalDate.now().toString()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_CLIENT, clientId); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_COUNT, 100L); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_LAST_MODIFIED, System.currentTimeMillis()); httpServletRequestMock.addHeader("Authorization", "TEST_TOKEN"); when(orcidTokenStoreMock.readClientId(eq("TEST_TOKEN"))).thenReturn(clientId); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByClientIdAndRequestDate(eq(clientId), any())).thenReturn(e); + when(papiRateLimitRedisMock.getDailyLimitsForClient(eq(clientId), any())).thenReturn(dailyLimitsObj); httpServletRequestMock.addHeader("X-REAL-IP", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); - verify(papiRateLimitingDaoMock, times(1)).updatePublicApiDailyRateLimit(any(PublicApiDailyRateLimitEntity.class), eq(true)); - verify(papiRateLimitingDaoMock, never()).persist(any(PublicApiDailyRateLimitEntity.class)); + verify(papiRateLimitRedisMock, times(1)).setTodayLimitsForClient(anyString(), any(JSONObject.class)); } @Test - public void doFilterInternal_checkLimitReachedTest() throws ServletException, IOException { + public void doFilterInternal_checkLimitReachedTest() throws ServletException, IOException, JSONException { MockitoAnnotations.initMocks(this); String ip = "127.0.0.2"; - PublicApiDailyRateLimitEntity e = new PublicApiDailyRateLimitEntity(); - e.setId(1000L); - e.setIpAddress(ip); - e.setRequestCount(10001L); + JSONObject dailyLimitsObj = new JSONObject(); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_DATE_CREATED, System.currentTimeMillis()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_IS_ANONYMOUS, true); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_DATE, LocalDate.now().toString()); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_CLIENT, ip); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_REQUEST_COUNT, 100000001L); + dailyLimitsObj.put(PapiRateLimitRedisClient.KEY_LAST_MODIFIED, System.currentTimeMillis()); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "enableRateLimiting", true); TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "orcidTokenStore", orcidTokenStoreMock); - TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRateLimitingDao", papiRateLimitingDaoMock); + TargetProxyHelper.injectIntoProxy(apiRateLimitFilter, "papiRedisClient", papiRateLimitRedisMock); - when(papiRateLimitingDaoMock.findByIpAddressAndRequestDate(eq(ip), any())).thenReturn(e); + when(papiRateLimitRedisMock.getTodayDailyLimitsForClient(eq(ip))).thenReturn(dailyLimitsObj); httpServletRequestMock.addHeader("X-REAL-IP", ip); apiRateLimitFilter.doFilterInternal(httpServletRequestMock, httpServletResponseMock, filterChainMock); assertEquals(429, httpServletResponseMock.getStatus()); String content = httpServletResponseMock.getContentAsString(); - assertEquals("Too Many Requests. You have exceeded the daily quota for anonymous usage of this API. \nYou can increase your daily quota by registering for and using Public API client credentials (https://info.orcid.org/documentation/integration-guide/registering-a-public-api-client/)", content); + assertEquals( + "Too Many Requests. You have exceeded the daily quota for anonymous usage of this API. \nYou can increase your daily quota by registering for and using Public API client credentials (https://info.orcid.org/documentation/integration-guide/registering-a-public-api-client/)", + content); } } \ No newline at end of file diff --git a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java index 73b1b4258c1..bf121b4b254 100644 --- a/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java +++ b/orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java @@ -4,6 +4,7 @@ import javax.annotation.Resource; +import org.orcid.core.api.rate_limit.PapiRateLimitRedisClient; import org.orcid.core.togglz.Features; import org.orcid.core.togglz.OrcidTogglzConfiguration; import org.orcid.persistence.dao.PublicApiDailyRateLimitDao; @@ -48,8 +49,11 @@ public class PapiDailyLimitReport { @Autowired private PublicApiDailyRateLimitDao papiRateLimitingDao; + + @Resource + private PapiRateLimitRedisClient papiRedisClient; - // for running spam manually + // for running manually public static void main(String[] args) { PapiDailyLimitReport dailyLimitReport = new PapiDailyLimitReport(); try { @@ -69,7 +73,13 @@ public static void main(String[] args) { public void papiDailyLimitReport() { LOG .info("start papi limit report the rate limiting is: " + enableRateLimiting); if (enableRateLimiting) { + try + { LocalDate yesterday = LocalDate.now().minusDays(1); + // save redis cached data to DB + papiRedisClient.saveRedisPapiLimitDateToDB(yesterday); + + //Report to Slack Channel String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING"; String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\nCurrent Anonymous Requests Limit: " + anonymousRequestLimit + "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\nMode: " + mode; @@ -81,6 +91,10 @@ public void papiDailyLimitReport() { + papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit); LOG .info(SLACK_STATS_MSG); slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl); + } + catch (Exception ex) { + slackManager.sendAlert("!!!!! Exception when storing papi limit redis data to DB. Check the logs" + "\n" + ex.toString() , slackChannel, webhookUrl, webhookUrl); + } } } diff --git a/orcid-test/src/main/resources/properties/test-core.properties b/orcid-test/src/main/resources/properties/test-core.properties index 1dc1d3e9298..666ac35657c 100644 --- a/orcid-test/src/main/resources/properties/test-core.properties +++ b/orcid-test/src/main/resources/properties/test-core.properties @@ -88,4 +88,12 @@ org.orcid.core.utils.panoply.password=xxx org.orcid.core.utils.panoply.idleConnectionTimeout=3600 org.orcid.core.utils.panoply.connectionTimeout=36000 org.orcid.core.utils.panoply.jdbcUrl=xxx -org.orcid.core.utils.panoply.username=xxx \ No newline at end of file +org.orcid.core.utils.panoply.username=xxx + + +# Papi Limits Redis Cache +org.orcid.core.utils.cache.papi.redis.host=xxx.xxx.com +org.orcid.core.utils.cache.papi.redis.port=6379 +org.orcid.core.utils.cache.papi.redis.password=XXXX +org.orcid.core.utils.cache.papi.redis.enabled=true + diff --git a/properties/development.properties b/properties/development.properties index 036dd07a4ae..d9fc7fb4644 100644 --- a/properties/development.properties +++ b/properties/development.properties @@ -277,7 +277,13 @@ org.orcid.core.utils.panoply.username=xxx #Slack channel for papi limits org.orcid.core.papiLimitReport.slackChannel=system-alerts-qa org.orcid.core.papiLimitReport.webhookUrl=xxx -org.orcid.papi.rate.limit.anonymous.requests=1 -org.orcid.papi.rate.limit.known.requests=2 +org.orcid.papi.rate.limit.anonymous.requests=1000 +org.orcid.papi.rate.limit.known.requests=11111 org.orcid.papi.rate.limit.enabled=true -org.orcid.scheduler.papiLimitReport.process=0 18 15 * * * \ No newline at end of file +org.orcid.scheduler.papiLimitReport.process=0 18 15 * * * + +# Papi Limits Redis Cache +org.orcid.core.utils.cache.papi.redis.host=xxxx +org.orcid.core.utils.cache.papi.redis.port=6379 +org.orcid.core.utils.cache.papi.redis.password=xxxx +org.orcid.core.utils.cache.papi.redis.enabled=true \ No newline at end of file