Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added the check for existent entries #7189

Merged
merged 1 commit into from
Jan 23, 2025
Merged

Added the check for existent entries #7189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 21 additions & 2 deletions orcid-core/src/main/java/org/orcid/core/api/rate_limit/PapiRateLimitRedisClient.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,9 @@ public class PapiRateLimitRedisClient {
@Value("${org.orcid.papi.rate.limit.redisCacheExpiryInSec:172800}")
private int CASH_EXPIRY_IN_SECONDS; // caching for 2 days to have time to
// synch with DB

@Value("${org.orcid.papi.rate.limit.checkExistentEntries:false}")
private boolean CHECK_DUPLICATES ; // if the scheduled job is less than one day, or we run the job twice we need to check for duplicates.

@Autowired
private PublicApiDailyRateLimitDao papiRateLimitingDao;
Expand Down Expand Up @@ -71,8 +74,24 @@ public void setTodayLimitsForClient(String client, JSONObject metaData) {
public void saveRedisPapiLimitDateToDB(LocalDate requestDate) throws JSONException {
// returns all the keys for requestDate
HashMap<String, JSONObject> allValuesForKey = redisClient.getAllValuesForKeyPattern("*" + requestDate.toString());
for (String key : allValuesForKey.keySet()) {
papiRateLimitingDao.persist(redisObjJsonToEntity(allValuesForKey.get(key)));
for (String key : allValuesForKey.keySet()) {
PublicApiDailyRateLimitEntity redisRateLimitEntity = redisObjJsonToEntity(allValuesForKey.get(key));
PublicApiDailyRateLimitEntity pgRateLimitEntity = null;
boolean isClient = false;
if(CHECK_DUPLICATES) {
if(StringUtils.isNotEmpty(redisRateLimitEntity.getIpAddress())) {
pgRateLimitEntity = papiRateLimitingDao.findByIpAddressAndRequestDate(redisRateLimitEntity.getIpAddress(), requestDate);
}
else if(StringUtils.isNotEmpty(redisRateLimitEntity.getClientId())){
pgRateLimitEntity = papiRateLimitingDao.findByClientIdAndRequestDate(redisRateLimitEntity.getClientId(), requestDate);
isClient = true;
}
}
if(pgRateLimitEntity != null) {
papiRateLimitingDao.updatePublicApiDailyRateLimit(pgRateLimitEntity, isClient);
} else {
papiRateLimitingDao.persist(redisObjJsonToEntity(allValuesForKey.get(key)));
}
redisClient.remove(key);
}
}
Expand Down
33 changes: 17 additions & 16 deletions orcid-scheduler-web/src/main/java/org/orcid/scheduler/report/PapiDailyLimitReport.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,22 +75,23 @@ public void papiDailyLimitReport() {
if (enableRateLimiting) {
try
{
LocalDate yesterday = LocalDate.now().minusDays(1);
// save redis cached data to DB
papiRedisClient.saveRedisPapiLimitDateToDB(yesterday);

//Report to Slack Channel
String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING";
String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\nCurrent Anonymous Requests Limit: " + anonymousRequestLimit
+ "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\nMode: " + mode;
LOG .info(SLACK_INTRO_MSG);
slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl);

String SLACK_STATS_MSG = "Count of Anonymous IPs blocked: " + papiRateLimitingDao.countAnonymousRequestsWithLimitExceeded(yesterday, anonymousRequestLimit)
+ "\nCount of Public API clients that have exceeded the limit: "
+ papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit);
LOG .info(SLACK_STATS_MSG);
slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl);
LocalDate yesterday = LocalDate.now().minusDays(1);
// save redis cached data to DB
papiRedisClient.saveRedisPapiLimitDateToDB(yesterday);
String SLACK_DB_MSG = "Public Api Stats from Redis saved in Postgres for the request date: " + yesterday.toString();
slackManager.sendAlert(SLACK_DB_MSG, slackChannel, webhookUrl, webhookUrl);
//Report to Slack Channel
String mode = Features.ENABLE_PAPI_RATE_LIMITING.isActive() ? "ENFORCEMENT" : "MONITORING";
String SLACK_INTRO_MSG = "Public API Rate limit report - Date: " + yesterday.toString() + "\nCurrent Anonymous Requests Limit: " + anonymousRequestLimit
+ "\nCurrent Public API Clients Limit: " + knownRequestLimit + "\nMode: " + mode;
LOG .info(SLACK_INTRO_MSG);
slackManager.sendAlert(SLACK_INTRO_MSG, slackChannel, webhookUrl, webhookUrl);

String SLACK_STATS_MSG = "Count of Anonymous IPs blocked: " + papiRateLimitingDao.countAnonymousRequestsWithLimitExceeded(yesterday, anonymousRequestLimit)
+ "\nCount of Public API clients that have exceeded the limit: "
+ papiRateLimitingDao.countClientRequestsWithLimitExceeded(yesterday, knownRequestLimit);
LOG .info(SLACK_STATS_MSG);
slackManager.sendAlert(SLACK_STATS_MSG, slackChannel, webhookUrl, webhookUrl);
}
catch (Exception ex) {
slackManager.sendAlert("!!!!! Exception when storing papi limit redis data to DB. Check the logs" + "\n" + ex.toString() , slackChannel, webhookUrl, webhookUrl);
Expand Down
Loading