Skip to content

Commit

Permalink
Merge pull request #8 from diging/develop
Browse files Browse the repository at this point in the history
Prepare release
  • Loading branch information
jdamerow authored Apr 13, 2017
2 parents 3b17671 + 1c72fc9 commit a90f75e
Show file tree
Hide file tree
Showing 16 changed files with 418 additions and 73 deletions.
4 changes: 2 additions & 2 deletions cassiopeia/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@
<org.slf4j-version>1.7.5</org.slf4j-version>
<org.springframework-version>4.3.1.RELEASE</org.springframework-version>
<spring-security-version>4.1.3.RELEASE</spring-security-version>
<geco.requests.version>0.2.1</geco.requests.version>
<geco.util.version>0.4.1</geco.util.version>
<geco.requests.version>0.4</geco.requests.version>
<geco.util.version>0.4.2</geco.util.version>

<tomcat.deploy.path></tomcat.deploy.path>
<tomcat.app.name>cassiopeia</tomcat.app.name>
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package edu.asu.diging.gilesecosystem.cassiopeia.core.service;

import edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.RequestInfo;

public interface IKafkaRequestSender {

public abstract void sendRequest(String requestId, String documentId,
RequestInfo info);

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package edu.asu.diging.gilesecosystem.cassiopeia.core.service;

import java.util.concurrent.ExecutionException;

import edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.ResendingResult;

public interface IRequestManager {

public abstract void startResendingRequests();

public abstract ResendingResult getResendingResults() throws InterruptedException,
ExecutionException;

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package edu.asu.diging.gilesecosystem.cassiopeia.core.service;

import java.util.concurrent.Future;

import edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.ResendingResult;

public interface IRequestResender {

public Future<ResendingResult> resendRequests();

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
package edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl;

import java.time.OffsetDateTime;
import java.time.ZoneId;

import javax.annotation.PostConstruct;

import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import edu.asu.diging.gilesecosystem.cassiopeia.core.properties.Properties;
import edu.asu.diging.gilesecosystem.cassiopeia.core.service.IKafkaRequestSender;
import edu.asu.diging.gilesecosystem.cassiopeia.rest.DownloadFileController;
import edu.asu.diging.gilesecosystem.requests.ICompletedOCRRequest;
import edu.asu.diging.gilesecosystem.requests.IRequestFactory;
import edu.asu.diging.gilesecosystem.requests.RequestStatus;
import edu.asu.diging.gilesecosystem.requests.exceptions.MessageCreationException;
import edu.asu.diging.gilesecosystem.requests.impl.CompletedOCRRequest;
import edu.asu.diging.gilesecosystem.requests.kafka.IRequestProducer;
import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;

@Service
public class KafkaRequestSender implements IKafkaRequestSender {

private final Logger logger = LoggerFactory.getLogger(getClass());

@Autowired
private IPropertiesManager propertyManager;

@Autowired
private IRequestFactory<ICompletedOCRRequest, CompletedOCRRequest> requestFactory;

@Autowired
private IRequestProducer requestProducer;


@PostConstruct
public void init() {
requestFactory.config(CompletedOCRRequest.class);
}

/* (non-Javadoc)
* @see edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.IKafkaRequestSender#sendRequest(java.lang.String, java.lang.String, java.lang.String, java.lang.String, edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.RequestInfo)
*/
@Override
public void sendRequest(String requestId, String documentId, RequestInfo info) {
String restEndpoint = propertyManager.getProperty(Properties.BASE_URL);
if (restEndpoint.endsWith("/")) {
restEndpoint = restEndpoint.substring(0, restEndpoint.length()-1);
}

String fileEndpoint = restEndpoint + DownloadFileController.GET_FILE_URL
.replace(DownloadFileController.REQUEST_ID_PLACEHOLDER, requestId)
.replace(DownloadFileController.DOCUMENT_ID_PLACEHOLDER, documentId)
.replace(DownloadFileController.FILENAME_PLACEHOLDER, info.getFilename());

ICompletedOCRRequest completedRequest = null;
try {
completedRequest = requestFactory.createRequest(requestId, info.getUploadId());
} catch (InstantiationException | IllegalAccessException e) {
logger.error("Could not create request.", e);
// this should never happen if used correctly
}

completedRequest.setDocumentId(documentId);
completedRequest.setDownloadPath(info.getPath());
completedRequest.setSize(info.getSize());
completedRequest.setDownloadUrl(fileEndpoint);
completedRequest.setFilename(info.getImageFilename());
completedRequest.setFileId(info.getFileId());
completedRequest.setStatus(RequestStatus.COMPLETE);
completedRequest.setOcrDate(OffsetDateTime.now(ZoneId.of("UTC")).toString());
completedRequest.setTextFilename(info.getFilename());

try {
requestProducer.sendRequest(completedRequest, propertyManager.getProperty(Properties.KAFKA_TOPIC_OCR_COMPLETE));
} catch (MessageCreationException e) {
logger.error("Could not send message.", e);
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,8 @@
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.util.Arrays;

import javax.annotation.PostConstruct;

import org.apache.tika.exception.TikaException;
import org.apache.tika.metadata.Metadata;
import org.apache.tika.parser.ParseContext;
Expand All @@ -34,15 +30,9 @@
import org.xml.sax.SAXException;

import edu.asu.diging.gilesecosystem.cassiopeia.core.properties.Properties;
import edu.asu.diging.gilesecosystem.cassiopeia.core.service.IKafkaRequestSender;
import edu.asu.diging.gilesecosystem.cassiopeia.core.service.IOCRManager;
import edu.asu.diging.gilesecosystem.cassiopeia.rest.DownloadFileController;
import edu.asu.diging.gilesecosystem.requests.ICompletedOCRRequest;
import edu.asu.diging.gilesecosystem.requests.IOCRRequest;
import edu.asu.diging.gilesecosystem.requests.IRequestFactory;
import edu.asu.diging.gilesecosystem.requests.RequestStatus;
import edu.asu.diging.gilesecosystem.requests.exceptions.MessageCreationException;
import edu.asu.diging.gilesecosystem.requests.impl.CompletedOCRRequest;
import edu.asu.diging.gilesecosystem.requests.kafka.IRequestProducer;
import edu.asu.diging.gilesecosystem.util.files.IFileStorageManager;
import edu.asu.diging.gilesecosystem.util.properties.IPropertiesManager;

Expand All @@ -59,24 +49,16 @@ public class OCRManager implements IOCRManager {
private IPropertiesManager propertyManager;

@Autowired
private IRequestFactory<ICompletedOCRRequest, CompletedOCRRequest> requestFactory;

@Autowired
private IRequestProducer requestProducer;


@PostConstruct
public void init() {
requestFactory.config(CompletedOCRRequest.class);
}
private IKafkaRequestSender kafkaRequestSender;


/* (non-Javadoc)
* @see edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.IOCRManager#processOCRRequest(edu.asu.diging.gilesecosystem.requests.IOCRRequest)
*/
@Override
public void processOCRRequest(IOCRRequest request) {

String dirFolder = storageManager.getAndCreateStoragePath(request.getRequestId(),
storageManager.getAndCreateStoragePath(request.getRequestId(),
request.getDocumentId(), null);
byte[] image = downloadFile(request.getDownloadUrl());

Expand All @@ -103,41 +85,11 @@ public void processOCRRequest(IOCRRequest request) {
// FIXME: send to monitoring app
}

Text text = saveTextToFile(request.getRequestId(), request.getDocumentId(), ocrResult, request.getFilename(), ".txt");

String restEndpoint = propertyManager.getProperty(Properties.BASE_URL);
if (restEndpoint.endsWith("/")) {
restEndpoint = restEndpoint.substring(0, restEndpoint.length()-1);
}

String fileEndpoint = restEndpoint + DownloadFileController.GET_FILE_URL
.replace(DownloadFileController.REQUEST_ID_PLACEHOLDER, request.getRequestId())
.replace(DownloadFileController.DOCUMENT_ID_PLACEHOLDER, request.getDocumentId())
.replace(DownloadFileController.FILENAME_PLACEHOLDER, text.filename);
RequestInfo info = saveTextToFile(request.getRequestId(), request.getDocumentId(), ocrResult, request.getFilename(), ".txt");
info.setUploadId(request.getUploadId());
info.setFileId(request.getFileId());

ICompletedOCRRequest completedRequest = null;
try {
completedRequest = requestFactory.createRequest(request.getRequestId(), request.getUploadId());
} catch (InstantiationException | IllegalAccessException e) {
logger.error("Could not create request.", e);
// this should never happen if used correctly
}

completedRequest.setDocumentId(request.getDocumentId());
completedRequest.setDownloadPath(text.path);
completedRequest.setSize(text.size);
completedRequest.setDownloadUrl(fileEndpoint);
completedRequest.setFilename(request.getFilename());
completedRequest.setFileId(request.getFileId());
completedRequest.setStatus(RequestStatus.COMPLETE);
completedRequest.setOcrDate(OffsetDateTime.now(ZoneId.of("UTC")).toString());
completedRequest.setTextFilename(text.filename);

try {
requestProducer.sendRequest(completedRequest, propertyManager.getProperty(Properties.KAFKA_TOPIC_OCR_COMPLETE));
} catch (MessageCreationException e) {
logger.error("Could not send message.", e);
}
kafkaRequestSender.sendRequest(request.getRequestId(), request.getDocumentId(), info);
}

private byte[] downloadFile(String url) {
Expand All @@ -161,14 +113,15 @@ private byte[] downloadFile(String url) {
return null;
}

protected Text saveTextToFile(String requestId,
protected RequestInfo saveTextToFile(String requestId,
String documentId, String pageText, String filename, String fileExtentions) {
String docFolder = storageManager.getAndCreateStoragePath(requestId,
documentId, null);

if (!fileExtentions.startsWith(".")) {
fileExtentions = "." + fileExtentions;
}
String imageFilename = filename;
filename = filename + fileExtentions;

String filePath = docFolder + File.separator + filename;
Expand All @@ -192,19 +145,6 @@ protected Text saveTextToFile(String requestId,
}

String relativePath = storageManager.getFileFolderPathInBaseFolder(requestId, documentId, null);
Text text = new Text(relativePath + File.separator + filename, fileObject.length(), filename);
return text;
}

class Text {
public String path;
public long size;
public String filename;

public Text(String path, long size, String filename) {
this.path = path;
this.size = size;
this.filename = filename;
}
return new RequestInfo(relativePath + File.separator + filename, fileObject.length(), imageFilename, filename);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
package edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl;

public class RequestInfo {

private String path;
private long size;
private String filename;
private String imageFilename;
private String uploadId;
private String fileId;

public RequestInfo(String path, long size, String imageFilename, String filename) {
this.path = path;
this.size = size;
this.filename = filename;
this.imageFilename = imageFilename;
}

public String getPath() {
return path;
}

public void setPath(String path) {
this.path = path;
}

public long getSize() {
return size;
}

public void setSize(long size) {
this.size = size;
}

public String getImageFilename() {
return imageFilename;
}

public void setImageFilename(String filename) {
this.imageFilename = filename;
}

public String getUploadId() {
return uploadId;
}

public void setUploadId(String uploadId) {
this.uploadId = uploadId;
}

public String getFileId() {
return fileId;
}

public void setFileId(String fileId) {
this.fileId = fileId;
}

public String getFilename() {
return filename;
}

public void setFilename(String filename) {
this.filename = filename;
}


}
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl;

import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;

import edu.asu.diging.gilesecosystem.cassiopeia.core.service.IRequestManager;
import edu.asu.diging.gilesecosystem.cassiopeia.core.service.IRequestResender;

@Service
public class RequestManager implements IRequestManager {

@Autowired
private IRequestResender resender;

private Future<ResendingResult> resendingResult;

/* (non-Javadoc)
* @see edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.IRequestManager#startResendingRequests()
*/
@Override
public void startResendingRequests() {
resendingResult = resender.resendRequests();
}

/* (non-Javadoc)
* @see edu.asu.diging.gilesecosystem.cassiopeia.core.service.impl.IRequestManager#getResendingResults()
*/
@Override
public ResendingResult getResendingResults() throws InterruptedException, ExecutionException {
if (resendingResult == null) {
return new ResendingResult(0, null);
}
if (resendingResult.isDone()) {
return resendingResult.get();
}

return null;
}
}
Loading

0 comments on commit a90f75e

Please sign in to comment.