Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main'
Browse files Browse the repository at this point in the history
  • Loading branch information
mdavis95 committed Mar 23, 2022
2 parents da6bef4 + cd6f478 commit 58eee77
Showing 1 changed file with 15 additions and 5 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.bson.Document;
import org.xerial.snappy.SnappyInputStream;
import org.xerial.snappy.SnappyOutputStream;
import software.amazon.awssdk.auth.credentials.*;
import software.amazon.awssdk.core.ResponseInputStream;
import software.amazon.awssdk.core.sync.RequestBody;
import software.amazon.awssdk.regions.Region;
Expand Down Expand Up @@ -68,7 +69,17 @@ public S3DocumentStorage(MongoClient mongoClient, String indexName, String dbNam
this.indexName = indexName;
this.dbName = dbName;
this.sharded = sharded;
this.s3 = S3Client.builder().region(Region.of(this.region)).build();
AwsCredentialsProviderChain credentialsProvider = AwsCredentialsProviderChain.builder()
.credentialsProviders(
InstanceProfileCredentialsProvider.builder().build(),
ContainerCredentialsProvider.builder().build(),
EnvironmentVariableCredentialsProvider.create(),
SystemPropertyCredentialsProvider.create(),
ProfileCredentialsProvider.builder().build()
)
.build();

this.s3 = S3Client.builder().region(Region.of(this.region)).credentialsProvider(credentialsProvider).build();

ForkJoinPool.commonPool().execute(() -> {
MongoDatabase db = client.getDatabase(dbName);
Expand Down Expand Up @@ -107,7 +118,7 @@ public void storeAssociatedDocument(AssociatedDocument doc) throws Exception {

PutObjectRequest req = PutObjectRequest.builder().bucket(bucket).key(key).contentLength((long) bytes.length).build();

ByteArrayOutputStream baos = new ByteArrayOutputStream(bytes.length);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
SnappyOutputStream os = new SnappyOutputStream(baos);
os.write(bytes);
os.flush();
Expand Down Expand Up @@ -154,8 +165,7 @@ public void getAssociatedDocuments(Writer outputstream, Document filter) throws
for (Document doc : found) {
if (first) {
first = false;
}
else {
} else {
outputstream.write(",\n");
}

Expand Down Expand Up @@ -334,7 +344,7 @@ private AssociatedDocument parseTOC(Document doc) throws IOException {
GetObjectRequest gor = GetObjectRequest.builder()
.bucket(s3Info.getString("bucket"))
.key(s3Info.getString("key"))
.build() ;
.build();
ResponseInputStream<GetObjectResponse> results = s3.getObject(gor);
InputStream compression = new SnappyInputStream(results);
try (compression) {
Expand Down

0 comments on commit 58eee77

Please sign in to comment.