Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

RateLimitig with Redis Implementation #7188

Merged
merged 2 commits into from
Jan 22, 2025
Merged

RateLimitig with Redis Implementation #7188

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f04a273
RateLimitig with Redis Implementation
Camelia-Orcid Jan 22, 2025
906ddc6
Merge branch 'main' into 9566-techrd-refactor-the-papi-quota-logic-so…
leomendoza123 Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 95 additions & 0 deletions orcid-core/src/main/java/org/orcid/core/api/rate_limit/PapiRateLimitRedisClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
package org.orcid.core.api.rate_limit;

import java.time.LocalDate;
import java.util.Date;
import java.util.HashMap;

import javax.annotation.Resource;

import org.apache.commons.lang.StringUtils;
import org.orcid.core.utils.cache.redis.RedisClient;
import org.orcid.persistence.dao.PublicApiDailyRateLimitDao;
import org.orcid.persistence.jpa.entities.PublicApiDailyRateLimitEntity;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;
import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;

@Component
public class PapiRateLimitRedisClient {
private static final Logger LOG = LoggerFactory.getLogger(PapiRateLimitRedisClient.class);

@Resource(name = "redisClientPapi")
private RedisClient redisClient;

@Value("${org.orcid.papi.rate.limit.redisCacheExpiryInSec:172800}")
private int CASH_EXPIRY_IN_SECONDS; // caching for 2 days to have time to
// synch with DB

@Autowired
private PublicApiDailyRateLimitDao papiRateLimitingDao;

public static final String KEY_REQUEST_COUNT = "reqCount";
public static final String KEY_REQUEST_DATE = "reqDate";
public static final String KEY_LAST_MODIFIED = "lastModified";
public static final String KEY_DATE_CREATED = "dateCreated";
public static final String KEY_IS_ANONYMOUS = "isAnonymous";
public static final String KEY_REQUEST_CLIENT = "reqClient";

public static final String KEY_DELIMITATOR = "||";

public String getTodayKeyByClient(String client) {
return getRequestDateKeyByClient(client, LocalDate.now());
}

public String getRequestDateKeyByClient(String client, LocalDate requestDate) {
return client + KEY_DELIMITATOR + requestDate.toString() ;
}

public JSONObject getDailyLimitsForClient(String client, LocalDate requestDate) {

String metaData = redisClient.get(getRequestDateKeyByClient(client, requestDate));
try {
return StringUtils.isNotBlank(metaData) ? new JSONObject(metaData) : null;
} catch (Exception ex) {
return null;
}
}

public JSONObject getTodayDailyLimitsForClient(String client) {
return getDailyLimitsForClient(client, LocalDate.now());
}

public void setTodayLimitsForClient(String client, JSONObject metaData) {
String limitKey = getTodayKeyByClient(client);
redisClient.set(limitKey, metaData.toString(), CASH_EXPIRY_IN_SECONDS);
}

public void saveRedisPapiLimitDateToDB(LocalDate requestDate) throws JSONException {
// returns all the keys for requestDate
HashMap<String, JSONObject> allValuesForKey = redisClient.getAllValuesForKeyPattern("*" + requestDate.toString());
for (String key : allValuesForKey.keySet()) {
papiRateLimitingDao.persist(redisObjJsonToEntity(allValuesForKey.get(key)));
redisClient.remove(key);
}
}

private PublicApiDailyRateLimitEntity redisObjJsonToEntity(JSONObject redisObj) throws JSONException {
PublicApiDailyRateLimitEntity rateLimitEntity = new PublicApiDailyRateLimitEntity();
if (!redisObj.getBoolean(KEY_IS_ANONYMOUS)) {
rateLimitEntity.setClientId(redisObj.getString(KEY_REQUEST_CLIENT));
} else {
rateLimitEntity.setIpAddress(redisObj.getString(KEY_REQUEST_CLIENT));
}
rateLimitEntity.setRequestCount(redisObj.getLong(KEY_REQUEST_COUNT));
rateLimitEntity.setRequestDate(LocalDate.parse(redisObj.getString(KEY_REQUEST_DATE)));
rateLimitEntity.setDateCreated(new Date(redisObj.getInt(KEY_DATE_CREATED)));
rateLimitEntity.setLastModified(new Date(redisObj.getInt(KEY_LAST_MODIFIED)));
papiRateLimitingDao.persist(rateLimitEntity);
return rateLimitEntity;
}

}
97 changes: 68 additions & 29 deletions orcid-core/src/main/java/org/orcid/core/utils/cache/redis/RedisClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,14 @@
import java.net.InetAddress;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.util.HashMap;
import java.util.List;

import javax.annotation.PostConstruct;
import javax.annotation.Resource;

import org.codehaus.jettison.json.JSONException;
import org.codehaus.jettison.json.JSONObject;
import org.orcid.utils.alerting.SlackManager;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand All @@ -17,43 +21,46 @@
import redis.clients.jedis.Jedis;
import redis.clients.jedis.JedisClientConfig;
import redis.clients.jedis.JedisPool;
import redis.clients.jedis.params.ScanParams;
import redis.clients.jedis.params.SetParams;
import redis.clients.jedis.resps.ScanResult;

public class RedisClient {

private static final Logger LOG = LoggerFactory.getLogger(RedisClient.class);

private static final int DEFAULT_CACHE_EXPIRY = 60;
private static final int DEFAULT_TIMEOUT = 10000;

public static final int MACH_KEY_BATCH_SIZE = 1000;

private final String redisHost;
private final int redisPort;
private final String redisPassword;
private final int cacheExpiryInSecs;
private final int clientTimeoutInMillis;
private JedisPool pool;
private SetParams defaultSetParams;

@Resource
private SlackManager slackManager;
// Assume the connection to Redis is disabled by default

// Assume the connection to Redis is disabled by default
private boolean enabled = false;

public RedisClient(String redisHost, int redisPort, String password) {
this.redisHost = redisHost;
this.redisPort = redisPort;
this.redisPassword = password;
this.cacheExpiryInSecs = DEFAULT_CACHE_EXPIRY;
this.clientTimeoutInMillis = DEFAULT_TIMEOUT;
this.clientTimeoutInMillis = DEFAULT_TIMEOUT;
}

public RedisClient(String redisHost, int redisPort, String password, int cacheExpiryInSecs) {
this.redisHost = redisHost;
this.redisPort = redisPort;
this.redisPassword = password;
this.cacheExpiryInSecs = cacheExpiryInSecs;
this.clientTimeoutInMillis = DEFAULT_TIMEOUT;
this.clientTimeoutInMillis = DEFAULT_TIMEOUT;
}

public RedisClient(String redisHost, int redisPort, String password, int cacheExpiryInSecs, int clientTimeoutInMillis) {
Expand All @@ -68,47 +75,48 @@ public RedisClient(String redisHost, int redisPort, String password, int cacheEx
private void init() {
try {
JedisClientConfig config = DefaultJedisClientConfig.builder().connectionTimeoutMillis(this.clientTimeoutInMillis).timeoutMillis(this.clientTimeoutInMillis)
.socketTimeoutMillis(this.clientTimeoutInMillis).password(this.redisPassword).ssl(true).build();
pool = new JedisPool(new HostAndPort(this.redisHost, this.redisPort), config);
defaultSetParams = new SetParams().ex(this.cacheExpiryInSecs);
.socketTimeoutMillis(this.clientTimeoutInMillis).password(this.redisPassword).ssl(true).build();
pool = new JedisPool(new HostAndPort(this.redisHost, this.redisPort), config);
defaultSetParams = new SetParams().ex(this.cacheExpiryInSecs);
// Pool test
try(Jedis jedis = pool.getResource()) {
if(jedis.isConnected()) {
try (Jedis jedis = pool.getResource()) {
if (jedis.isConnected()) {
LOG.info("Connected to the Redis cache, elements will be cached for " + this.cacheExpiryInSecs + " seconds");
// As it was possible to make the connection, enable the client
// As it was possible to make the connection, enable the
// client
enabled = true;
}
}
} catch(Exception e) {
} catch (Exception e) {
LOG.error("Exception initializing Redis client", e);
try {
// Lets try to get the host name
InetAddress id = InetAddress.getLocalHost();
// Lets try to get the host name
InetAddress id = InetAddress.getLocalHost();
slackManager.sendSystemAlert("Unable to start Redis client on " + id.getHostName());
} catch(UnknownHostException uhe) {
} catch (UnknownHostException uhe) {
// Lets try to get the IP address
try(final DatagramSocket socket = new DatagramSocket()){
try (final DatagramSocket socket = new DatagramSocket()) {
socket.connect(InetAddress.getByName("8.8.8.8"), 10002);
String ip = socket.getLocalAddress().getHostAddress();
slackManager.sendSystemAlert("Unable to start Redis client on IP " + ip);
} catch(SocketException | UnknownHostException se) {
slackManager.sendSystemAlert("Unable to start Redis client - Couldn't identify the machine");
}
} catch (SocketException | UnknownHostException se) {
slackManager.sendSystemAlert("Unable to start Redis client - Couldn't identify the machine");
}
}
}
}

public boolean set(String key, String value) {
return set(key, value, defaultSetParams);
}

public boolean set(String key, String value, int cacheExpiryInSecs) {
SetParams params = new SetParams().ex(cacheExpiryInSecs);
return set(key, value, params);
}

private boolean set(String key, String value, SetParams params) {
if(enabled && pool != null) {
if (enabled && pool != null) {
try (Jedis jedis = pool.getResource()) {
LOG.debug("Setting Key: {}", key);
String result = jedis.set(key, value, params);
Expand All @@ -117,17 +125,17 @@ private boolean set(String key, String value, SetParams params) {
}
return false;
}

public String get(String key) {
if(enabled && pool != null) {
if (enabled && pool != null) {
try (Jedis jedis = pool.getResource()) {
LOG.debug("Reading Key: {}" , key);
LOG.debug("Reading Key: {}", key);
return jedis.get(key);
}
}
}
return null;
}

public boolean remove(String key) {
if (enabled && pool != null) {
try (Jedis jedis = pool.getResource()) {
Expand All @@ -141,4 +149,35 @@ public boolean remove(String key) {
}
return true;
}

/**
* Retrieve the mapped key, value for all the keys that match the matchKey
* parameter.
*
* @param machKey
* the key pattern for which the mapped values are returned
* @return
* @throws JSONException
*/
public HashMap<String, JSONObject> getAllValuesForKeyPattern(String matchKey) throws JSONException {
HashMap<String, JSONObject> mappedValuesForKey = new HashMap<String, JSONObject>();
// Connect to Redis
try (Jedis jedis = pool.getResource()) {
String cursor = "0";
ScanParams scanParams = new ScanParams().match(matchKey).count(MACH_KEY_BATCH_SIZE);
do {
// Use SCAN to fetch matching keys in batches
ScanResult<String> scanResult = jedis.scan(cursor, scanParams);
cursor = scanResult.getCursor();
List<String> keys = scanResult.getResult();

// Print each key and its corresponding value
for (String key : keys) {
mappedValuesForKey.put(key, new JSONObject(jedis.get(key)));
}
} while (!"0".equals(cursor)); // SCAN ends when cursor returns "0"
}

return mappedValuesForKey;
}
}
12 changes: 12 additions & 0 deletions orcid-core/src/main/resources/orcid-core-context.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1228,6 +1228,18 @@
<constructor-arg index="3" value="${org.orcid.core.utils.cache.redis.expiration_in_secs:600}" />
<constructor-arg index="4" value="${org.orcid.core.utils.cache.redis.connection_timeout_millis:10000}" />
</bean>


<!-- Redis cache -->
<bean id="redisClientPapi" class="org.orcid.core.utils.cache.redis.RedisClient">
<constructor-arg index="0" value="${org.orcid.core.utils.cache.papi.redis.host}"/>
<constructor-arg index="1" value="${org.orcid.core.utils.cache.papi.redis.port}" />
<constructor-arg index="2" value="${org.orcid.core.utils.cache.papi.redis.password}" />
<constructor-arg index="3" value="${org.orcid.core.utils.cache.papi.redis.expiration_in_secs:600}" />
<constructor-arg index="4" value="${org.orcid.core.utils.cache.papi.redis.connection_timeout_millis:10000}" />
</bean>

<bean id="papiRateLimitRedisClient" class="org.orcid.core.api.rate_limit.PapiRateLimitRedisClient"/>

<bean id="eventManager" class="org.orcid.core.common.manager.impl.EventManagerImpl"/>

Expand Down
4 changes: 2 additions & 2 deletions ...tence/src/main/java/org/orcid/persistence/jpa/entities/PublicApiDailyRateLimitEntity.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,15 +93,15 @@ public Date getDateCreated() {
return dateCreated;
}

void setDateCreated(Date date) {
public void setDateCreated(Date date) {
this.dateCreated = date;
}

public Date getLastModified() {
return lastModified;
}

void setLastModified(Date lastModified) {
public void setLastModified(Date lastModified) {
this.lastModified = lastModified;
}

Expand Down
Loading
Loading