Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Cloud storage SDK Upgrade #92

Open
wants to merge 5 commits into
base: release-5.1.0
Choose a base branch
from
Open

Cloud storage SDK Upgrade #92

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9a99dff
#0 feat: Upgrading the cloud storage sdk
manjudr Nov 23, 2022
1c5f5f9
#OB-45 feat: Updated the cloud storage sdk
manjudr Nov 30, 2022
334b4ca
#OB-45 feat: Run tests
manjudr Nov 30, 2022
d039af2
#OB-45 feat: Run tests
manjudr Nov 30, 2022
a3e636e
#OB-45 feat: Generation of the signed url for the relative path
manjudr Dec 15, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions analytics-api-core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -221,8 +221,8 @@
</dependency>
<dependency>
<groupId>org.sunbird</groupId>
<artifactId>cloud-store-sdk</artifactId>
<version>1.4.0</version>
<artifactId>cloud-store-sdk_${scala.maj.version}</artifactId>
<version>1.4.4</version>
<exclusions>
<exclusion>
<groupId>com.microsoft.azure</groupId>
Expand Down
9 changes: 6 additions & 3 deletions analytics-api-core/src/main/scala/org/ekstep/analytics/api/service/JobAPIService.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -267,7 +267,7 @@ class JobAPIService @Inject()(postgresDBUtil: PostgresDBUtil, apiValidator: APIV
}
else {
val expiry = config.getInt("channel.data_exhaust.expiryMins")
(dateKey, storageService.getSignedURL(bucket, key, Option((expiry * 60))))
(dateKey, storageService.getSignedURLV2(bucket, key, Option((expiry * 60))))
}
}
return (isValid, res)
Expand Down Expand Up @@ -377,12 +377,15 @@ class JobAPIService @Inject()(postgresDBUtil: PostgresDBUtil, apiValidator: APIV
val values = f.split("/").toList.drop(4) // 4 - is derived from 2 -> '//' after http, 1 -> uri and 1 -> container
values.mkString("/")
}
else {
else if(f.contains("wasb")) {
val values = f.split("/").toList.drop(3) // 3 - is derived from 2 -> '//' after wasb, 1 -> uri
values.mkString("/")
} else{
val values = f.split("/").toList
values.mkString("/")
}
APILogger.log("Getting signed URL for - " + objectKey)
storageService.getSignedURL(bucket, objectKey, Option((expiry * 60)))
storageService.getSignedURLV2(bucket, objectKey, Option((expiry * 60)))
} else List[String]()
JobResponse(job.request_id, job.tag, job.job_id, job.requested_by, job.requested_channel, job.status, lastupdated, request, job.iteration.getOrElse(0), stats, Option(downloadUrls), Option(Long.box(expiryTime)), job.err_message)
}
Expand Down
58 changes: 58 additions & 0 deletions analytics-api-core/src/test/scala/org/ekstep/analytics/api/service/TestJobAPIService.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -751,4 +751,62 @@ class TestJobAPIService extends BaseSpec {
values.mkString("/") should be("uci-response-exhaust/427C56316649909179E69188C5CDB091/d655cf03-1f6f-4510-acf6-d3f51b488a5e_response_20220209.csv")

}


it should "return signed URL when url is absolute path" in {

EmbeddedPostgresql.execute(
s"""truncate table job_request;""")
EmbeddedPostgresql.execute(
s"""insert into job_request ("tag", "request_id", "job_id", "status", "request_data", "requested_by",
"requested_channel", "dt_job_submitted", "dt_job_completed", "download_urls", "dt_file_created", "execution_time") values ('client-2', '462CDD1241226D5CA2E777DA522691EF', 'assessment-score-report',
'SUCCESS', '{"batchFilter":["TPD","NCFCOPY"],"contentFilters":{"request":{"filters":{"identifier":["do_11305960936384921612216","do_1130934466492252161819"],"prevState":"Draft"},"sort_by":{"createdOn":"desc"},"limit":10000,"fields":["framework","identifier","name","channel","prevState"]}},"reportPath":"course-progress-v2/"}',
'test-1', 'in.ekstep' , '2020-09-07T13:54:39.019+05:30', '2020-09-08T13:54:39.019+05:30', '{"wasb://[email protected]/uci-response-exhaust/427C56316649909179E69188C5CDB091/d655cf03-1f6f-4510-acf6-d3f51b488a5e_response_20220209.csv"}', '2020-09-08T13:50:39.019+05:30', '10');""")

when(cacheUtil.getConsumerChannelTable()).thenReturn(mockTable)
when(mockTable.get(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1)
reset(mockStorageService)
when(mockFc.getStorageService(ArgumentMatchers.any(),ArgumentMatchers.any(),ArgumentMatchers.any())).thenReturn(mockStorageService);
when(mockStorageService.getSignedURLV2(ArgumentMatchers.any(),ArgumentMatchers.any(),ArgumentMatchers.any(),ArgumentMatchers.any())).thenReturn("https://sunbird.org/test/signed/file1.csv");
doNothing().when(mockStorageService).closeContext()

val res = jobApiServiceActorRef.underlyingActor.getDataRequestList("client-2", 10, requestHeaderData)
val resultMap = res.result.get
val jobRes = JSONUtils.deserialize[List[JobResponse]](JSONUtils.serialize(resultMap.get("jobs").get))
jobRes.length should be(1)

// check for extracting object key logic
val testStr = "wasb://[email protected]/uci-response-exhaust/427C56316649909179E69188C5CDB091/d655cf03-1f6f-4510-acf6-d3f51b488a5e_response_20220209.csv"
val values = testStr.split("/").toList.drop(3)
values.mkString("/") should be("uci-response-exhaust/427C56316649909179E69188C5CDB091/d655cf03-1f6f-4510-acf6-d3f51b488a5e_response_20220209.csv")

}

it should "return signed URL when url is relative path" in {

EmbeddedPostgresql.execute(
s"""truncate table job_request;""")
EmbeddedPostgresql.execute(
s"""insert into job_request ("tag", "request_id", "job_id", "status", "request_data", "requested_by",
"requested_channel", "dt_job_submitted", "dt_job_completed", "download_urls", "dt_file_created", "execution_time") values ('client-2', '462CDD1241226D5CA2E777DA522691EF', 'assessment-score-report',
'SUCCESS', '{"batchFilter":["TPD","NCFCOPY"],"contentFilters":{"request":{"filters":{"identifier":["do_11305960936384921612216","do_1130934466492252161819"],"prevState":"Draft"},"sort_by":{"createdOn":"desc"},"limit":10000,"fields":["framework","identifier","name","channel","prevState"]}},"reportPath":"course-progress-v2/"}',
'test-1', 'in.ekstep' , '2020-09-07T13:54:39.019+05:30', '2020-09-08T13:54:39.019+05:30', '{"progress-exhaust/778A3669531143769B5E8C98D42E1CAD/013307481768378368112_progress_20220927.zip"}', '2020-09-08T13:50:39.019+05:30', '10');""")

when(cacheUtil.getConsumerChannelTable()).thenReturn(mockTable)
when(mockTable.get(ArgumentMatchers.any(), ArgumentMatchers.any())).thenReturn(1)
reset(mockStorageService)
when(mockFc.getStorageService(ArgumentMatchers.any(),ArgumentMatchers.any(),ArgumentMatchers.any())).thenReturn(mockStorageService);
when(mockStorageService.getSignedURLV2(ArgumentMatchers.any(),ArgumentMatchers.any(),ArgumentMatchers.any(),ArgumentMatchers.any())).thenReturn("https://sunbird.org/test/signed/013307481768378368112_progress_20220927.zip");
doNothing().when(mockStorageService).closeContext()

val res = jobApiServiceActorRef.underlyingActor.getDataRequestList("client-2", 10, requestHeaderData)
val resultMap = res.result.get
val jobRes = JSONUtils.deserialize[List[JobResponse]](JSONUtils.serialize(resultMap.get("jobs").get))
jobRes.length should be(1)

// check for extracting object key logic
val testStr = "progress-exhaust/778A3669531143769B5E8C98D42E1CAD/013307481768378368112_progress_20220927.zip"
val values = testStr.split("/").toList
values.mkString("/") should be("progress-exhaust/778A3669531143769B5E8C98D42E1CAD/013307481768378368112_progress_20220927.zip")
}
}