From 34a2453dbc9757fadffb5249761849b9d87d35d8 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Oct 2018 09:20:08 -0400 Subject: [PATCH 001/157] typos, fix for empty string --- .../iq/dataverse/DataverseServiceBean.java | 10 ++-- .../edu/harvard/iq/dataverse/api/Admin.java | 16 ++++--- .../command/impl/CreateDataverseCommand.java | 46 ++++++++++--------- 3 files changed, 38 insertions(+), 34 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 6da6eb13a19..1447318a52c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -649,7 +649,7 @@ public List findAllDataverseDatasetChildren(Long dvId) { } } - public JsonObjectBuilder addRoleAssignementsToChildren(Dataverse owner, ArrayList rolesToInherit, + public JsonObjectBuilder addRoleAssignmentsToChildren(Dataverse owner, ArrayList rolesToInherit, boolean inheritAllRoles) { /* * This query recursively finds all Dataverses that are inside/children of the @@ -778,9 +778,9 @@ public JsonObjectBuilder addRoleAssignementsToChildren(Dataverse owner, ArrayLis * handled. Add this to the log and the API return message. */ logger.info(Json.createObjectBuilder().add("Dataverses Updated", dataverseIds) - .add("Updated Dataverse Aliases", dataverseAliases).add("Admins added", usedNames) - .add("Admins not added", unusedNames).build().toString()); - return Json.createObjectBuilder().add("Dataverses Updated", dataverseAliases).add("Admins added", usedNames) - .add("Admins not added", unusedNames); + .add("Updated Dataverse Aliases", dataverseAliases).add("Assignments added for", usedNames) + .add("Assignments not added for", unusedNames).build().toString()); + return Json.createObjectBuilder().add("Dataverses Updated", dataverseAliases).add("Assignments added for", usedNames) + .add("Assignments not added for", unusedNames); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index e2ce08de20a..3b430463378 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1301,15 +1301,17 @@ public Response addRoleAssignementsToChildren(@PathParam("alias") String alias) } boolean inheritAllRoles = false; String rolesString = settingsSvc.getValueForKey(SettingsServiceBean.Key.InheritParentRoleAssignments, ""); - ArrayList rolesToInherit = new ArrayList(Arrays.asList(rolesString.split("\\s*,\\s*"))); - if (!rolesToInherit.isEmpty()) { - if (rolesToInherit.contains("*")) { - inheritAllRoles = true; + if (rolesString.length() > 0) { + ArrayList rolesToInherit = new ArrayList(Arrays.asList(rolesString.split("\\s*,\\s*"))); + if (!rolesToInherit.isEmpty()) { + if (rolesToInherit.contains("*")) { + inheritAllRoles = true; + } + + return ok(dataverseSvc.addRoleAssignmentsToChildren(owner, rolesToInherit, inheritAllRoles)); } - - return ok(dataverseSvc.addRoleAssignementsToChildren(owner, rolesToInherit, inheritAllRoles)); } return error(Response.Status.BAD_REQUEST, - "InheritParentRoleAssignements does not list any roles on this instance"); + "InheritParentRoleAssignments does not list any roles on this instance"); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java index a1bc3ff54fe..7bcf645ff91 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java @@ -107,30 +107,32 @@ public Dataverse execute(CommandContext ctxt) throws CommandException { boolean inheritAllRoles = false; String rolesString = ctxt.settings().getValueForKey(SettingsServiceBean.Key.InheritParentRoleAssignments, ""); ArrayList rolesToInherit = new ArrayList(Arrays.asList(rolesString.split("\\s*,\\s*"))); - if (!rolesToInherit.isEmpty()) { - if (rolesToInherit.contains("*")) { - inheritAllRoles = true; - } + if (rolesString.length() > 0) { + if (!rolesToInherit.isEmpty()) { + if (rolesToInherit.contains("*")) { + inheritAllRoles = true; + } - List assignedRoles = ctxt.roles().directRoleAssignments(owner); - for (RoleAssignment role : assignedRoles) { - // If all roles are to be inherited, or this role is in the list, and, in both - // cases, this is not an admin role for the current user which was just created - // above... - if ((inheritAllRoles || rolesToInherit.contains(role.getRole().getAlias())) - && !(role.getAssigneeIdentifier().equals(getRequest().getUser().getIdentifier()) - && role.getRole().equals(adminRole))) { - String identifier = role.getAssigneeIdentifier(); - if (identifier.startsWith(AuthenticatedUser.IDENTIFIER_PREFIX)) { - identifier = identifier.substring(AuthenticatedUser.IDENTIFIER_PREFIX.length()); - ctxt.roles().save(new RoleAssignment(role.getRole(), - ctxt.authentication().getAuthenticatedUser(identifier), managedDv, privateUrlToken)); - } else if (identifier.startsWith(Group.IDENTIFIER_PREFIX)) { - identifier = identifier.substring(Group.IDENTIFIER_PREFIX.length()); - Group roleGroup = ctxt.groups().getGroup(identifier); - if (roleGroup!=null) { + List assignedRoles = ctxt.roles().directRoleAssignments(owner); + for (RoleAssignment role : assignedRoles) { + // If all roles are to be inherited, or this role is in the list, and, in both + // cases, this is not an admin role for the current user which was just created + // above... + if ((inheritAllRoles || rolesToInherit.contains(role.getRole().getAlias())) + && !(role.getAssigneeIdentifier().equals(getRequest().getUser().getIdentifier()) + && role.getRole().equals(adminRole))) { + String identifier = role.getAssigneeIdentifier(); + if (identifier.startsWith(AuthenticatedUser.IDENTIFIER_PREFIX)) { + identifier = identifier.substring(AuthenticatedUser.IDENTIFIER_PREFIX.length()); ctxt.roles().save(new RoleAssignment(role.getRole(), - roleGroup, managedDv, privateUrlToken)); + ctxt.authentication().getAuthenticatedUser(identifier), managedDv, privateUrlToken)); + } else if (identifier.startsWith(Group.IDENTIFIER_PREFIX)) { + identifier = identifier.substring(Group.IDENTIFIER_PREFIX.length()); + Group roleGroup = ctxt.groups().getGroup(identifier); + if (roleGroup != null) { + ctxt.roles().save(new RoleAssignment(role.getRole(), + roleGroup, managedDv, privateUrlToken)); + } } } } From e353ccc5555d457e3be9d5d7a01b5334fd23cc5a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Oct 2018 12:42:06 -0400 Subject: [PATCH 002/157] use String return value the JsonObjectBuilders were getting cleared in the build().toString() used to generate the log message, so the returned value was mostly empty... --- .../edu/harvard/iq/dataverse/DataverseServiceBean.java | 10 +++++----- src/main/java/edu/harvard/iq/dataverse/api/Admin.java | 1 - 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index 1447318a52c..d53caf989ad 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -649,7 +649,7 @@ public List findAllDataverseDatasetChildren(Long dvId) { } } - public JsonObjectBuilder addRoleAssignmentsToChildren(Dataverse owner, ArrayList rolesToInherit, + public String addRoleAssignmentsToChildren(Dataverse owner, ArrayList rolesToInherit, boolean inheritAllRoles) { /* * This query recursively finds all Dataverses that are inside/children of the @@ -777,10 +777,10 @@ public JsonObjectBuilder addRoleAssignmentsToChildren(Dataverse owner, ArrayList * entities that had an admin role on the specified dataverse which were not * handled. Add this to the log and the API return message. */ - logger.info(Json.createObjectBuilder().add("Dataverses Updated", dataverseIds) + String result = Json.createObjectBuilder().add("Dataverses Updated", dataverseIds) .add("Updated Dataverse Aliases", dataverseAliases).add("Assignments added for", usedNames) - .add("Assignments not added for", unusedNames).build().toString()); - return Json.createObjectBuilder().add("Dataverses Updated", dataverseAliases).add("Assignments added for", usedNames) - .add("Assignments not added for", unusedNames); + .add("Assignments not added for", unusedNames).build().toString(); + logger.info(result); + return (result); } } diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index 3b430463378..2d15439b6ce 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1307,7 +1307,6 @@ public Response addRoleAssignementsToChildren(@PathParam("alias") String alias) if (rolesToInherit.contains("*")) { inheritAllRoles = true; } - return ok(dataverseSvc.addRoleAssignmentsToChildren(owner, rolesToInherit, inheritAllRoles)); } } From 7a0f302300c3f3ab5e757dcefe84fb090dd73bc3 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 16 Oct 2018 15:29:51 -0400 Subject: [PATCH 003/157] filter custom roles --- .../iq/dataverse/DataverseServiceBean.java | 5 ++- .../command/impl/CreateDataverseCommand.java | 35 ++++++++++--------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java index d53caf989ad..ac564e2915c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataverseServiceBean.java @@ -699,7 +699,10 @@ public String addRoleAssignmentsToChildren(Dataverse owner, ArrayList ro List inheritableRAsOnOwner = new ArrayList(); for (RoleAssignment role : allRAsOnOwner) { if (inheritAllRoles || rolesToInherit.contains(role.getRole().getAlias())) { - inheritableRAsOnOwner.add(role); + //Only supporting built-in/non-dataverse-specific custom roles. Custom roles all have an owner. + if(role.getRole().getOwner()==null) { + inheritableRAsOnOwner.add(role); + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java index 7bcf645ff91..7a352590aac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateDataverseCommand.java @@ -115,23 +115,26 @@ public Dataverse execute(CommandContext ctxt) throws CommandException { List assignedRoles = ctxt.roles().directRoleAssignments(owner); for (RoleAssignment role : assignedRoles) { - // If all roles are to be inherited, or this role is in the list, and, in both - // cases, this is not an admin role for the current user which was just created - // above... - if ((inheritAllRoles || rolesToInherit.contains(role.getRole().getAlias())) - && !(role.getAssigneeIdentifier().equals(getRequest().getUser().getIdentifier()) - && role.getRole().equals(adminRole))) { - String identifier = role.getAssigneeIdentifier(); - if (identifier.startsWith(AuthenticatedUser.IDENTIFIER_PREFIX)) { - identifier = identifier.substring(AuthenticatedUser.IDENTIFIER_PREFIX.length()); - ctxt.roles().save(new RoleAssignment(role.getRole(), - ctxt.authentication().getAuthenticatedUser(identifier), managedDv, privateUrlToken)); - } else if (identifier.startsWith(Group.IDENTIFIER_PREFIX)) { - identifier = identifier.substring(Group.IDENTIFIER_PREFIX.length()); - Group roleGroup = ctxt.groups().getGroup(identifier); - if (roleGroup != null) { + //Only supporting built-in/non-dataverse-specific custom roles. Custom roles all have an owner. + if (role.getRole().getOwner() == null) { + // And... If all roles are to be inherited, or this role is in the list, and, in both + // cases, this is not an admin role for the current user which was just created + // above... + if ((inheritAllRoles || rolesToInherit.contains(role.getRole().getAlias())) + && !(role.getAssigneeIdentifier().equals(getRequest().getUser().getIdentifier()) + && role.getRole().equals(adminRole))) { + String identifier = role.getAssigneeIdentifier(); + if (identifier.startsWith(AuthenticatedUser.IDENTIFIER_PREFIX)) { + identifier = identifier.substring(AuthenticatedUser.IDENTIFIER_PREFIX.length()); ctxt.roles().save(new RoleAssignment(role.getRole(), - roleGroup, managedDv, privateUrlToken)); + ctxt.authentication().getAuthenticatedUser(identifier), managedDv, privateUrlToken)); + } else if (identifier.startsWith(Group.IDENTIFIER_PREFIX)) { + identifier = identifier.substring(Group.IDENTIFIER_PREFIX.length()); + Group roleGroup = ctxt.groups().getGroup(identifier); + if (roleGroup != null) { + ctxt.roles().save(new RoleAssignment(role.getRole(), + roleGroup, managedDv, privateUrlToken)); + } } } } From 2910168fe6fbdb6ce0b88d0416f98466568ecfb7 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Mon, 12 Nov 2018 12:53:44 -0500 Subject: [PATCH 004/157] Don't open input stream when not needed. --- .../edu/harvard/iq/dataverse/api/Access.java | 14 +++--- .../dataaccess/ImageThumbConverter.java | 4 +- .../iq/dataverse/dataaccess/S3AccessIO.java | 43 +++++++++++++------ .../iq/dataverse/dataaccess/StorageIO.java | 4 +- .../dataverse/dataaccess/StorageIOTest.java | 4 +- 5 files changed, 43 insertions(+), 26 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Access.java b/src/main/java/edu/harvard/iq/dataverse/api/Access.java index 02f6f427027..019c9009f14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Access.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Access.java @@ -651,15 +651,14 @@ public InputStream fileCardImage(@PathParam("fileId") Long fileId, @Context UriI || "application/zipped-shapefile".equalsIgnoreCase(df.getContentType())) { thumbnailDataAccess = ImageThumbConverter.getImageThumbnailAsInputStream(dataAccess, 48); + if (thumbnailDataAccess != null && thumbnailDataAccess.getInputStream() != null) { + return thumbnailDataAccess.getInputStream(); + } } } } catch (IOException ioEx) { return null; } - - if (thumbnailDataAccess != null && thumbnailDataAccess.getInputStream() != null) { - return thumbnailDataAccess.getInputStream(); - } return null; } @@ -695,6 +694,9 @@ public InputStream dsCardImage(@PathParam("versionId") Long versionId, @Context dataAccess.open(); thumbnailDataAccess = ImageThumbConverter.getImageThumbnailAsInputStream(dataAccess, 48); } + if (thumbnailDataAccess != null && thumbnailDataAccess.getInputStream() != null) { + return thumbnailDataAccess.getInputStream(); + } } catch (IOException ioEx) { thumbnailDataAccess = null; } @@ -711,10 +713,6 @@ public InputStream dsCardImage(@PathParam("versionId") Long versionId, @Context } }*/ - if (thumbnailDataAccess != null && thumbnailDataAccess.getInputStream() != null) { - return thumbnailDataAccess.getInputStream(); - } - } return null; diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index a18af48cdd0..41860fb0c3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -271,12 +271,12 @@ private static boolean generateImageThumbnail(StorageIO storageIO, int try { storageIO.open(); + return generateImageThumbnailFromInputStream(storageIO, size, storageIO.getInputStream()); } catch (IOException ioex) { logger.warning("caught IOException trying to open an input stream for " + storageIO.getDataFile().getStorageIdentifier() + ioex); return false; } - - return generateImageThumbnailFromInputStream(storageIO, size, storageIO.getInputStream()); + } /* diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 530f7ee4a17..bbae7468286 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -12,6 +12,7 @@ import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; +import com.amazonaws.services.s3.model.GetObjectMetadataRequest; import com.amazonaws.services.s3.model.GetObjectRequest; import com.amazonaws.services.s3.model.ListObjectsRequest; import com.amazonaws.services.s3.model.ObjectListing; @@ -123,22 +124,13 @@ public void open(DataAccessOption... options) throws IOException { if (isReadAccess) { key = getMainFileKey(); - S3Object s3object = null; + ObjectMetadata objectMetadata = null; try { - s3object = s3.getObject(new GetObjectRequest(bucketName, key)); + objectMetadata = s3.getObjectMetadata(bucketName, key); } catch (SdkClientException sce) { throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); } - InputStream in = s3object.getObjectContent(); - - if (in == null) { - throw new IOException("Cannot get InputStream for S3 Object" + key); - } - - this.setInputStream(in); - - setChannel(Channels.newChannel(in)); - this.setSize(s3object.getObjectMetadata().getContentLength()); + this.setSize(objectMetadata.getContentLength()); if (dataFile.getContentType() != null && dataFile.getContentType().equals("text/tab-separated-values") @@ -181,6 +173,33 @@ public void open(DataAccessOption... options) throws IOException { } } + @Override + public InputStream getInputStream() throws IOException { + if(super.getInputStream()==null) { + try { + setInputStream(s3.getObject(new GetObjectRequest(bucketName, key)).getObjectContent()); + } catch (SdkClientException sce) { + throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); + } + } + + if (super.getInputStream() == null) { + throw new IOException("Cannot get InputStream for S3 Object" + key); + } + + setChannel(Channels.newChannel(super.getInputStream())); + + return super.getInputStream(); + } + + public Channel getChannel() throws IOException { + if(super.getChannel()==null) { + getInputStream(); + } + return channel; + } + + // StorageIO method for copying a local Path (for ex., a temp file), into this DataAccess location: @Override public void savePath(Path fileSystemPath) throws IOException { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java index fda93b3f557..99eb36d44b0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java @@ -227,7 +227,7 @@ public boolean canWrite() { // getters: - public Channel getChannel() { + public Channel getChannel() throws IOException { return channel; } @@ -276,7 +276,7 @@ public long getSize() { return size; } - public InputStream getInputStream() { + public InputStream getInputStream() throws IOException { return in; } diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java index 70dbc11db29..f85b59053f9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java @@ -32,7 +32,7 @@ public class StorageIOTest { StorageIO instance = new FileAccessIO<>(); @Test - public void testGetChannel() throws FileNotFoundException { + public void testGetChannel() throws IOException { assertEquals(null, instance.getChannel()); Channel c = new RandomAccessFile("src/main/java/Bundle.properties", "r").getChannel(); instance.setChannel(c); @@ -104,7 +104,7 @@ public void testSize() { } @Test - public void testInputStream() { + public void testInputStream() throws IOException { assertEquals(null, instance.getInputStream()); InputStream is = new ByteArrayInputStream("Test".getBytes()); instance.setInputStream(is); From 7ed30fb11e419c051c003749820208e895bcc5ba Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 13 Nov 2018 10:12:24 -0500 Subject: [PATCH 005/157] simplify --- .../iq/dataverse/dataaccess/ImageThumbConverter.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java index 41860fb0c3e..f273d215a3e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/ImageThumbConverter.java @@ -146,12 +146,11 @@ public static InputStreamIO getImageThumbnailAsInputStream(StorageIO s try { storageIO.open(); - Channel cachedThumbnailChannel = storageIO.openAuxChannel(THUMBNAIL_SUFFIX + size); - if (cachedThumbnailChannel == null) { - logger.warning("Null channel for aux object " + THUMBNAIL_SUFFIX + size); + cachedThumbnailInputStream = storageIO.getAuxFileAsInputStream(THUMBNAIL_SUFFIX + size); + if (cachedThumbnailInputStream == null) { + logger.warning("Null stream for aux object " + THUMBNAIL_SUFFIX + size); return null; } - cachedThumbnailInputStream = Channels.newInputStream((ReadableByteChannel) cachedThumbnailChannel); int cachedThumbnailSize = (int) storageIO.getAuxObjectSize(THUMBNAIL_SUFFIX + size); InputStreamIO inputStreamIO = new InputStreamIO(cachedThumbnailInputStream, cachedThumbnailSize); From 08a0e7e37d497bc7facbfd06d4e424e05c29fba4 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 13 Nov 2018 12:42:57 -0500 Subject: [PATCH 006/157] add @Override --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index bbae7468286..0bdcdfd6be4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -192,6 +192,7 @@ public InputStream getInputStream() throws IOException { return super.getInputStream(); } + @Override public Channel getChannel() throws IOException { if(super.getChannel()==null) { getInputStream(); From e5fed4aeaaffc20dc256004dda504d918bd6447a Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Tue, 13 Nov 2018 17:37:14 -0500 Subject: [PATCH 007/157] override getReadChannel as well --- .../edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 0bdcdfd6be4..616085fcf08 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -35,6 +35,7 @@ import java.net.URLEncoder; import java.nio.channels.Channel; import java.nio.channels.Channels; +import java.nio.channels.ReadableByteChannel; import java.nio.channels.WritableByteChannel; import java.nio.file.Path; import java.nio.file.Paths; @@ -199,8 +200,14 @@ public Channel getChannel() throws IOException { } return channel; } - + @Override + public ReadableByteChannel getReadChannel() throws IOException { + //Make sure StorageIO.channel variable exists + getChannel(); + return super.getReadChannel(); + } + // StorageIO method for copying a local Path (for ex., a temp file), into this DataAccess location: @Override public void savePath(Path fileSystemPath) throws IOException { From 6ea25243eafd0f76659490edad354f9e5f249376 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Fri, 21 Dec 2018 10:07:36 -0500 Subject: [PATCH 008/157] TDL image and release name changes --- pom.xml | 2 +- .../resources/images/favicondataverse.png | Bin 1184 -> 7165 bytes src/main/webapp/resources/images/tdl.png | Bin 0 -> 14054 bytes 3 files changed, 1 insertion(+), 1 deletion(-) create mode 100644 src/main/webapp/resources/images/tdl.png diff --git a/pom.xml b/pom.xml index 1692c190922..1368d2c23a4 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ --> edu.harvard.iq dataverse - 4.10 + 4.10-tdl war dataverse diff --git a/src/main/webapp/resources/images/favicondataverse.png b/src/main/webapp/resources/images/favicondataverse.png index dd1d8b3fb7e8395ec73a63cb4652de24bf17311a..1b6d61ad1b4c2929ac8b7d985d25f6d740fc25b8 100644 GIT binary patch literal 7165 zcmV=<5uKi000}DNklXnhZKXH)Lzi zbAXPgJz!~XK>$z!NcTNF`gtfC`)}CTdf55G}wm-7bfbZS=(0bsW?K_7c zATS5a=k@wO0btIVv5Ux#+JCU-K3lQk>L1X-0Jv zJ2&2S_W%fmx}`nFF8?Vc+`t4iGDhUr?^!=pjujs79d+;DHcc90PS7GL6KcuszF$)V zd8*8Y5Sh+cb!i&ll||v_B&vwBe^@K zfU9o5-O3KA@SGuovito9HaJ0tO9ovaPw&*yff@-S=D6e0t!t;I8;QXgNvEXo{&!Zb zZU6uVGXNI&bd<2q5BSg$+(q!6+wTiYmyNO&B1oyubsfX{ojZ39GTbxVKp31)>*B#( zRUsTYZo2c%95iS|wy0VZVYaTV?A);Wu1pGiUYW_KZFxxexYwxcx@}XpdKL>~b!yaW zQtIiRhnZsO#Sg99Mux^hvS~%$tshTQzs4(*S9XZ79FLc~za)x}RY!1G?mfXeZmHETx_Q9Ia5m~=L7g-U=_#xf8xgb9;gAcW`BDvc``i|uorRRAdI-_t~KoSW~(}e z=!64ChgpCS?(UXpvAlEb!<(x{R54#XJ$Zb~k5n?&f}Vd&BDcTr1zy){*EkCiKrNMm z*_3kVd|~zNW#|_yXOV`;3~H!KXSXg!L0veJg1c>^T({w|!w)++K+py!{zPfDKarOs zz!rws{1@$cp+O^n5IRRqz}u5)`z9eD{~$t0RzM;U05xJ*cRSsOCr9_K+qC4Fhossw z(|Cw^YNns-ZcZt%DL|^GA_-~3Yj3{pw4+X|eIz77*-QHf=spO(eM}20ZG*bZmm{qo z45jP5=;=^E>5&fVK4;jU;Colr002dTHz+#|apjG7#@>=qqamtV(rS3(vkrUh2`k6z zMp@OG-Gx?+#9jS>YPL*HC@7T2+gjd--}WbZN(T)tumZ~6Vr(Zcz~%9}?)WMAw1zN% zNREop2bLx!e&gQzi*ivmZJGqhMew`sSoMly4n5`gqZR`SBg~u)wK0qdSS2t*0DC+R zHC@0X5zv|-;d5&n0uW9M6%(iqnwg3KF7P&d@QI>fTfOcn?RU$($Z(yrsnpadcGnM^ z)f?97(w}PyOH-M#digUJfBK7GEW=x0zw%i<)WJrOKreby40Fn%>1eB^{P*|WclP=O zK8nbIWTU$~-GvDmuu%E{Cdq_azyX@Lfth<$+IM3k09by!MYrBQ7c{s;xWGFCz4oR% zI-Q9MS)6b)cZ6bRYPu^{oOki12fX`?mmYCYM`&KF8|z%sXNnln0FK8O4{UynO7ro$ zO)#X+ZvFuztVx zF2kQ*c**Tsc3E#soB>FR96fo8&T)y|%^N>{<~v?^_|jmNCLoni-&rzXkN!N)Gd&?< zd!iWBp4k8iFrWYmN4J4^=4Y?H;V2m{QDc`lI@Awdz6LTCF_!ie(oPLrgib;vxRVqSBfMQ{ zA_)mlV0Ijn7o7d><6R5$roh^e&iw2@uAi=aVtMu=dK9-5>MCSG2TtQ$7@RjYF-fAe z1c#F5o{}_~&7!7X1y-oX(s0GD9iMpn=`TEBQ4b^5b(#T&Q9t_kU;X0M*JJ5HQ!@$U zi;#zcc`3r_WN=}d3WmSDaQva)q}c- zTaG^O%E;ZK25Jd}t3wTJ-ziySS9j2xojz8>$KLdY6S}dR;n7E)IAeo8gO5hkP73k< znfed^?mBS0ry#MIrTq^N?Vs|aACsy?R-q=>e~ovA7S!EWJ$5C`VW zAgSagce}k|Gt<%fc}vGW{nlSSqyb=4QSTg#&iUKFpYBe~vP@C{YOGuF?n#;eBN_?8 zOg#&bNRt9ylIm&G;VS2ze(Fix&>tBrc@gU<`jHR)IU!VsW_s?H@r^Hj-h}Yl+wNAO z7KMpGf@>gyyYA(oTa+yz?$&>xjeL9jh=9_191^6ril3Yj1w2$y2f;UQ-8i{<%g-P6 ztbUo*n&q)L;n_d;jc%uoWsk&&$vY9tYa;o#WW>5u*9 zn~&+nF?ZDlQ7EdUe)vy6?4$`>Nfz*s@+rq2bNop!_~Fer4X39QkU@lwj|{OSvI9ao zd`@U2_xKTjbLIQ)X9}c)0(Z`dIk{VKk4=uLDm`g4vs`n(9vyx7AdkjNs0#a`@fb>g_m4<-K}}?fpzGcl-4B1rUjz(dDlfU_-=r-y=~YTK?XYM zLMWW#Lda-#Oo*A8mCt(C3s)ZVU!MCj2aomWxg;VaW$aYmHaYuOU;ECw$zifU*{#TP zSD#DjYvc~1npVgbb1~TUY~Z|io_=({=#xHZpq5KXphBaQ*$0HuBb?@LLI*&LHmp2W zTKwa!_y5VK|9(_1l5$a&m_rRhK%r}En!>#SQ@l+{S{tSWZ)e9X2JzfjQ>EGIl}9W) z`S{~r`Ml>H(~n~r;;>v<43%gNTz|)dU%KJ?o40LCi&ivsQL>vt1;XbZZ{CJI7t~tR zq6<7t9aZ{g&v@hE3|4xPEGCBu3Q2)&x?w%F{DAK3PdxsO$DQzu<$ZVx z)B?=OoDvipW=0p?c-z;mzjbRSmmW|T{b8!hpiHUoS`px((a=OXbFfremeiS*Bs~qo zT}L`T^;>WKnV=6ww%!Gdr;5%cp_ZARy3b z*$VMO4OVO5PWA#=WN-60Bh*A4gGEwr-Si8~mVWHlPd!3HaAa?qCp`hPO1wXgGVm^;5jQhMTgUL zLnT&D>{4|e7M$b?kygL&J4Zm^u~5&i#ii3^qJGlSW}JH)qg$d#s9o| zU8DrM!*W2PsHvLAGMSA}*$9oILIXhYKp_p)HB}_h01}Yp|k(MWcA+9 zes1+hLT@a_zG%_ZIfO`L1w$$_>(J+5cH20cmmPonJ750N(MUN~-%Gz;a}vQ7v^ue&%&XOvu^9!bf9 zx=D2!FUGR*vCXIb{0XO>@}i>-m`IIpbt!VC?un8BQo1mo-hRTNhyL}i{_=Qcr^8WG zK?PuhX(p5lqG?^=ZLeq8-4W>GS=`wvL)k|@Y1yK$p8H4ddf5w#sr5Zf!`#dabaR6{ z%nb>ufi}ef6fhU8Ay0aB+Ljt6pt8cJw^=V?fG*VmT{W&^uc-&mMNvoP7lt6R{;Ap$P7Q{;n8bw2Q zSJX``jn_q4g{og|e)+h5;!UquYE2!g z0um6Rgg}WB6hvs;7v(BHmic=ebH=%j9CtXf14!ylvx$xl>bg!jmfe(-yEzPIVy2-$ zV~dt7yYMUD7#7T#9VqChtbF!Cg?6@XI1&HeYT-ZgvTco&UH)G^?!Q(v`Z+tlk{b;^=x znujnSN!a3Y@s&UL`nRtMgXx+mLy@v|%Tn@8xoh@CM;`pAZ$Gt95Scl5q*!Q+#6kgM zj-w?UN!=jPRWJMQwchOyYts;{X$Trz4GK3zNyskE@}+-$`8WUeZEuWqMySTTeB(*S z3qWr)H#Li43|HOz@MpgKwMS>F-r{F4+0KkoJ*upI@F5xOdinF8|AEtAyVzZ&ayT`h z0y+y@h+6PsnzVW@ZJef=&H~@M^$rwWhk7%vD74 zb~Rz2;r7Xs@NV|Z8~}p@w>8FTZ)?ruvg#zM{8$X}7>x@>AU~y-c6YJ?ufE~7(CgQh=xE!?%v;LD zG1mw`EOK2oUNlrI2B!B-Pk;8R?_$N$Q4!$*1B)rZScwgTH92Q905R&ihu6|F;UVgb zL@x?!i6eJ1JF_pm6cTPIO=avHS_8*M7a-AVxWgixbps+_qF!^tJp)G0?EipL)5=PD{HVyKcx zT2iYsX@FTa_vDr-7JECgpZT#*fB6Tu%wjj^>+bNh@O?i&O;bzGnW0F;6*t{6D#m8( z1h|{E+B`=snzu!@Nk4~f3XWK_+VcDry-UyeA76R*ubuz4)5dmeDVixS>Y)rxT^j3F zLP$2F>L{npQ-c{`Tk^9$`sr=O*s2YiADL|$5OfOkN82Ly7hnoRkS!sxa;B5UwYgZ{`fu5?ZsYYzxk4t|Mcg7bdZcVwXut#jb?hW zBMlvHMD5mS7zW(|oc)P^xMRv^v_FaX{U6?w70x+gKP=qoE*NkIT$qV_AKUfNrmeO1 z;MBdEq+#9`7_AHIvI#pkAMMru`@Q#^f7-7s_f&Q@r9sE@%4NNOJ@1?|PI_LsX~Trs zV3Y^3Z`tXV+;i-#ovQxuZ!W!M^X&GqgKFs{jW^$QcQx1VL9y^t=;y+{Gyx#Hee-+Q z7ZZzflc)l-z1H)*HD9=^d)mI?lw%IN^nd-q>yA2naXspIYRu|>&ryv9Ut)IN>A(ER zv(N00hE1+#=*hD=%*mp>{M*;xw)T-~Z0S_&)1be7^T*%(;;)?Z*2DTC3K2R8WfbWE zS-OC9J@=FS%5g_r@)u{HxNO*;zB7ktwT&m?P*5%%#BmZ6?&7up8^yqq_bt=l|Y2Uia$8Qz4R4 zgSq`W>Y#xw9YeD|aah^;rw{zzdtd#UxcQOA7>%l3xgU18HChxF$0{_Vrp`9L?10O! z{=NZHb}S`bc+;LNRG*DL1O562q z9_h3sZDm8rpzm%Z0SPfQ^)lg-2LJQ5C;!9$JoA}(XVFa7&7fA+9*~2kLPMdUGi>~Z zTW+5Q5+N$u@=yB-FiAQBkic(VcY8u4*hokcjP5~0fnsLsGe_G$b@tmnfA*VK(E2{2 zRhpBEwA)tZxuyWf+`6+vkyF)!hYG)V#Ik?+i*sIfVZu&rUk^E$UztkP@mdy;xjtmz2HxN>%~Wo!3sB^ z43HoSGm3zFu}3!7GS>*3;9k&zX_>p;92DgXXP8DR@+|U{9bgd_KWK2PHc9c@= zU3u$WLn1@y`!f#DHFChOzWKiGxkT)UXSySeI?aqe_Q<9MQ3$1X$F8~k zuE!Fx1kd|9EIBblyW-YWHnA)Ln(Mx5oSHhw(`6rg@5kQy%JJ+f0tqJ3AaoG2`_;zbSQ zG{rqBfPGJ9mEeiYIoAgJ+BNG-=@!^V+to}x_Mu;Y&1D~W)ARZY6N$lLbo77@pdeKE z_&z!T1E?Kn(URuv0!Pb!6DlB}p$qB4wJ#pBo%##Uy5gh1fAUh6(+`o>^pCi8-DvHs znN2@wUYdL9u>)pcD9Lc?wYRb8#+|zkvHZ^;`M~eK;U$IT?6Z*FeJk*vq_vy>(PoL3 z7i<5)2hE?kD1QFK?>qY!PwqbUh}P4O?%ICOhDU21ds0=prMD>#%@T%=?X~@I-5R9f zYmYnbd!PQ`s}Eb&Z*pLTNMgR{;mO>mF}J8I2&k9j`rJ3a>Wk<6_F)`#Vspj!Zc5Yz zCzrmjVeX#WS7X+=&$xg7@~g*R`_8|*X7v{LoZW}DkA~*etk0U6KY7E<+6%S^*>ar+ z-pqPc50c+8sL%M+MJs>(%!eD#3%v4A1h{I_Ow-Jozvr9}-?nkb&TQGesfV6NW>v~o zC*L2#_8Q*gllc|gLu7pqlj9)%pONIN%o4hQ`-)oIAB4cmH*K`&b<4!%wUtfs{rW3 zn%$zN8MJJtMfUR+D{~_=K_Hk}uAAJ1PEQv&)BA3DXgPN`(BKX*yFc-mj#PnPRIHZt82`Ti~3Uk?B!Ylp0*+7m{3+ootz+WN)WnQ(*-(AUCxn zQK2F?C$HG5!d3}vt`(3C64qBz04piUwpD^SD#ABF!8yMuRl!uxKsVXI%s|1+P|wiV z#N6CmN5ROz&_Lh7NZ-&%*U;R`*vQJjKmiJrfVLH-q*(>IxIyg#@@$ndN=gc>^!3Zj z%k|2Q_413-^$jg8EkR}&8R-I5=oVMzl_XZ^<`pZ$OmImpPAEg{v+u2}(t{7puX=A(aKG z`a!A1`K3k4z=%sz23b{L-^Aq1JP;qO z-q+X4Gq1QLF)umQ)5TT^Xo6m5W{Q=evzxJ_fvc&jo1vSbp{uc@vyqF1vx$YJnUk@h zv6CT8uSMv>2~2MaLa!N4y`ZF!TL84#CABECEH%ZgC_h&L>}9J=+-@<(X&zK> z3U0TU;MA)Rbc{YIVv!;mCIn19ASOK70y*%6pPC0u?M1+3y=JB2N(Kf-5lvW%RhGCbK;l~bAv`t>epce+-*e0t;K z-1qPAi@7G2+?meY{eIvH?p>VGx3ayV{ey5OnS%X{5NNG68YFQ sW5%}3#U)8=Z)v1xS4_-0_G6OEK~S0D>FVdQ&MBb@0OwhkEC2ui diff --git a/src/main/webapp/resources/images/tdl.png b/src/main/webapp/resources/images/tdl.png new file mode 100644 index 0000000000000000000000000000000000000000..6de3ed12cb36db431e2144ea8e06279263b34322 GIT binary patch literal 14054 zcmeHuRa{%a_HLk1f)%$yOK^uEE$&j>-Gdc(cS?}r#odELi#xQqx40K~FYY%z_uO;- zFZba*+=u)7oyhF$%jKgYke!9l@&i=qP<1~005XW(h{oh<2L-Q2}FdK>Ro|kMKYYuYtax@2-dsB_tb8$?;+i z-};ScSn^K9!$jXsJQ&dVMf8cy_<;b5%*>k^kW;a=49D^;7bge1G_%>l8blMGMWMRZ zfZd3o$H}$qL8Em z5v0^sPzH%RIGcmGSh-liERvp9?i`dtXdnS+GYdXd38{aIfS(CcTDrPA^0BdbczCdS zaI!i$e`I6l<>h4qbFgu6u)tfexOmySLOfaQU8w$w_}?-l%w0^KtsGsg9PB}VWkO6G z+*}1IDdF{?e+vJL{=cN#yZjRbI2>$$YuMOX!EFDtxvQ1M|E2lgntwO{doiD~m8ZF_ zwuF_PxxEYg76d6dIKlrA_U~Gq|53~H{vVZp1@Xz)K|Y#mS(&+7{`;zbS4f*%eYAvk z+;x?yAjjYBg;@^e%dqt-^T+#R66R?%mm){gWzZ!5(v?s~ft7-n{>B5V= zSSbs4MZI7F@{=>3MURpdZTfIwe0nB7FEN2IsW6Y8quQ`$nOI>_$`}vt z%rP5QQp(NRYLXb`NZ~my^XO~O%rTDvKEynOTIok)v(O&#*Xyp?;th6v>>KF7oa^*M zaaN_jWfmA$y;=t51=-`B_x2Q`=S6OdJ%Ko^+^}wJ*Lu2 z9z>ud@|=HIarb^1_wKquz;U?h5YEsUlq@2&h?s)V6sH?>k>fzD1RQC2L1oNt!QMyw z7)C+5d8-s!4AoT51pw(W|7_Nxp7VZU?oSYnblGR_G0)OhO^*N0!835?$LXHI=DDr~ zGWn)YtvcL;>Rc+zXmToJQTdzX-c|lw7K1NXS12s-G_=d~{m;PpYE#EDpN03^r=Exu z0BQ@T7Ac+d^l#MM^>ewyGV0~jl&-#~-k;WDWaQ%bLu20{NNY<`K=3v>+Ep1LX=+v( z!{nK+cX#qvS;1-sY7G?SYE4byKdf{mODQ0bhCE7xoTS{Dp`Kl60a{jT1ym^GU|XS8 zYFQ`(8(9K}5(w zzd2s#{EipO(h@I|1o%ZYva_{Q?6fme(o;%!e<<1epKa`TxP9mpdcH*OtS)g?%+69? z5VF7ZXmv8}{}qI4v&$1BMOa`W-X*!kdoWx>v;&is?BCY0Gc6j<@eY{U7P`X@j?-k7 z(O{BQb*-NdO_C&emo~`kvQ+(g{pJN;a^p3r@bTMuVf(cKCvk;{_*Q?91Kc8??9CD_cIatm*{hyFek-D>UCoP@)g}`Su@~Ux#^U7%f z=j%1<(d&`6GN2K`j`~M(7C}GS51w8Y+G*r6xc3F)atO9K%S<<_kB!P`GxeC zv`^PHFbxtJqmfF%pkHjimv67LvVI$D(^_XjwP#d@6OR93cI`>)}|xuJ+Jt6g=T>@Nv{=iiz8 zc%BQfcl!oNCfztvzy*1ih%M$N#w%tb$>c!aR%+9; zhV^(EszG4o?HZHx#)#hcx|JV}Jfwvbfm zBVr_04m2#cnb>%NCSoPQECLW=M}8|ByDWHo`IFadTJ=yxy#bp*~@O%t1}NQ zrIqWas<5xR$(33;{B5B;PtgrQ+zT&1kr{nSzFAGeBuX^P8UmFL%iP{a6D?gTo^NN5 zr|5YPO-a@POyxm-uAh7@Nq6=pe{gm98Flm>3Zpb&3*;C{_?W?OIJX>)GxAgZ zX*ra}MwUaVEF(f^dnPo(EJ>f$f{lLkPK@FJ)9H;xD-$+h;A?h535gdXi7`#>QpJsw zqR~5vsMQ!_aub_0Z^RM!#1f>Z83}h_*>_3iCJh1#0V6q3uMWGZ2mk(G!g}7JcJ@E0 z6zKgrP3gH+&I7dGXY&ud9vv&r;Iq*GjZyjMOLMM3wHeTdO%n_N3LSHR&9N(r7>VxI*O}R`EW<*V{vA)+ zWINwTPV`-2)8uo)b{ojc%Eop0*aSDs#(~)VC#f}e9r!}XINeh_MN91sSAz#y8s^k} zB54APOJ_PKbG$7MsXHt7+zl8?!##&=Y`RTuGPusUNtV`$iF(0DQXXfps0(8Y|uY0h8}cFXnpUF7!G$Wd?(-hiUl!y8%@WwgZna13-H zWeyG?-*u;M8fU$OYd=MwSywIna?`EcjD3bg3v{tldEdRnI|y>V6aU(96^g18pW>~f z@q+|7iXo*n!loUUCeLt4&V53b$%prd5C^UX^W@jc^R$tB#3J!lu`W8A8VsTy8ttH) zbwdwdbNU45(SI~YFk_}N)gc)kAYnEo<{eIL3o^V(2K{Rv z^_T3gf4R;kM*rBoTUeKX9geW(f05~!R(=KGv^P3mRlm8IlnSk=f|R2p6xt+FzEqRF zAHDK3wTN2fA7-S^Z%z@|d5Q=EcbdB1#nDQsgzBCNYas7gH|K_+>Zc&$TPB{u3Tz1R z1v}iHOP29eGK1H$#3#*5?|3`Mw|}!|PuOOaanY>Vj)d3@*Ql#j(-9wMRr5dQO^S7n zl6qRYWK3_QL`uTfM%?5z;4&n09XP0{JiS$v9r-rfXW-SsvLNA3g%&0P%p#|=e_uN% zCAlK5D_&f>|vO^kwP^*RH%Xx*AS%}pxTb(jdmQyi7@&dDidvPut4 zK2!Ubt-)(luk4OUvEEkE+>0+oEq<)yX)Usqstu+qgabimL%6v+{pr@BXa8kjms#Zv zexbGps0YWJEKkx`K6rvmJXy7+Gynq$3lSEGWFOfkdOTePAsK$E>o^?7)Pah8P3&_Y z80X>f`4h(D0FJ^@m7_7avL%RDk5(0leWnk&F|vC+sUyYm=>O|+YElTMl0R-cf4W@h-e}>l!5e!uai7*@&Om8Pw5)!SbidjCQwy^_|Es5+=bK+ zVHD`nIGhO*z6>r39pDciKMc(C&mmd#sjcpPjwT%--{ujcW@eR>v1FF+tZJ~-f!yH) z622}<3|C}?oI@|tvg}N^shYAKv%Wjo<}w2+n1_}w9*0jBo9+v;RWgL783^<>@eyDm zILfjr7fcfFhy(&JaI}6LcFvmuB_9kEDWUYr5;&3<<1>Z~!}=PTG(^=a<*XN>w<$hS zS$T1eLNUJ)U!fQ0sk|aQHybZt?rUb8e-Sy9l8pjs!vBtAx+(!+=DZft%hIO=eA-~3 z_{dKSEj8*gVRtEfahQGSrZtO?nj7!V;Z5Mh^@uOOx>yb+y(2Y;EEcNa4x+wvf(`VfY^r<#20q<|Cr>)b&67#s#b zH8pp)&7EIF#HGZK*+oPH0;seTD;~Yv*5X}qlX4KwpcuNZNKU{~bgz>5Ogl=nq!sKt zM~P0uc&F!^J5X2@gVRkX|dnBtFa~P9;AmM z1tQYaGH0N9(z+(^q510EpmaG{=$LO};Qs)@pktnhvGwi&?>R;C|^1pkq4Zt8t;}gXZ$D`$VCpUVz60V z^5n)cJ>4iUM!2p?;?Z940p zIpDtK?nxBd5%wskL3)lmt9@Ms_yypld<;YJwQUvW@hOn?vwRkZsFQQ;RDHc9QzWP0 zk5jcpeyOA50!-R-&IE*9k3RPIHkm?UaRfcR)Ud;+$OEItyYzKxOsR0(TmUtq3W%%A zaFF?9bfB8ZyU*aY`C3ZShTx&hB{lqgN2o(c*|S02vo^LkStDKmzF%e zIHm17G@axLvHJD3&>`L6GPh;!7|tNxE;{yv6k}9of$V+vxTOIY8(w zr$sejDb2TBkr zLDEdhR@>?H^nomAk}A&UwHo|D2}$_cY?~-y9vqI1!No`njFn(Or2aV1T<_~7HQ`|$ z@m!s^7Nmg=2vpt<Q&eDC==mA_7g&!c_$#-FPsL@zofmN;p)`Zz67|WfsnnDiSAD zGVaG38$1-rIp)Jj9TzXF;!saI}b%30V8ZPMmwzj@3R`H%CkLTs zk3^sZ0)R!@;CHCRG=ML>>6Cy7uxy;ner-IFIq6WmtBx*Akrsy;BpP!9<`*K#v`wpv z;%UE94$rB#TGL6Gp`ao{7707lk(CAADUB6yOy zNtq-^N<%%VpoH7RRTX0ayPP|}RVMLr8lSf(ddJm)?8ExHU+(Q&&o;8hwe;>M>WBDe zr|7p?`7bw>=<%v;&^Xa9_WK*ieXHwd$=AUl;b)P`5eW3yP%ttgKE1IBM(DcP(YvJE z$Qn$N*CA2n1wXrSYJ8R+bo-SL5_<`OC{T|mY%$QBLkRJyP7~V9th72QqVAWHCrLk@ zLH+*FoJ8ig&uWWU9V_hkPL{}BgkJ-?j?)~y4?A+{h+mN#76^!!w|V&ax%J5u&T(H~ zUZ*MI?<7^J_*Yg*u;9HANu|r+3u`U1!@RY+681|sKdVHc@N9M!EX{~8n3YQwUg^>c9jFY z(xsgG)6=g|hZ867Q|wSSO|E>H%FJB2yyw+OHR8&(p3mZYZ$GDhXgJPszo!RK0DytO zjs)Nk5CO^nD3eiYE8-AN5ADSFCmZT#`%LKK!dC?Elht3S ziY24(ZA(TphI#GOs=q=J^{a*2FD)w0L zKk2=kN3|YzdblT3I(%9!$T{+{Jns-mon(T=8nm1AT7`z8Cl+X})OI}Gb^7sGItg~x zwlWGbK;9psQq&buAN-tU?%eBD8ns_(KRF@u@bs@~p;ah1jy>t6N%Z~%DQ9zB*9|U4 z@MVv5Fyi>iq41m$dqtL6bJvv#s>c;$e~B1~0AIAYFbd751lCVSICqh*zRmO8I!^vT zXpK}w1HQWL14;t1`&5$$m6>KXVbg`Q4^AIGbdJ8MPD21DQ4tnW3w*12?{bW$F~VC- zOv7j%>7sVh@~wfjAKdskBA9TemY*-ggJkt)#eTuh#Xg_(XlQY(r$o+qoL0=O*d) zboVDS`b{_72jL>2V=Enq&U$41%DT$1^!(!(9xB7rTNK`Z-CSK&SPc@?p^1DSRm|0c z<3NA-3ffX^_iRtTw8XiM9@@*?{r2PcoVQ;sz`DsPch?76g7pRG(e4|C!ymAXWf?>R zKQ03S(7^G1+c(@qG}aOr7U2BEg-3+0Gaywi5F44=vlTniDMr%^BHGt0Ndo)4U|5bW znS}$S01)O)>o5rG37oX`mZw6TFZS>ByOC*hwA6{{$3!)7GPsGEeCAA+P%!#tndyhd z%X94a=ce8*3EH(fY*aljwl#7|=2WTdZmLFoKgLqzZ-2B1>%{{lS{JTQ)1*Dudj3wI zU_P_5V0asY@&I?J+~jq`DylxF^I6x%Pu2}-f4MHZ4bkD7hRo`!Q=&`r;gMLLVa+|m*HMapkT}=WkPF@i21*fjkpAu~YQ;Y!(*8YACYk8GN z_q5DVRU`lseT*~^A4I&pvPZ0|m1(6s1Qv5klQEZBu=@?bnX~TUK(cy$OK|SYHlV~N ztvou!hvTD(kICescC63Ersbk}muyI3kZKpM!lXf$Fg=%Vmeq^LQ!S8E5uZZrx@C3u ziQ3LEpHGy6pWR_pZmRG7MxnaNFVr|jIR)+XoCX)i2<0K;i|Lkls%$oL0?0=QB|X&bO~bHZO?Sua6&?8q_;=_7?2 zy=iW|p4h>P(r6#IQN!jr)C<3b$B=%uku-0x|U8nbF8j1iR>N(jLiyH5mb*FU{Yei#X z75Wn*C5Q0-h>4Z*W505y8&VVpFbBh!nQ+*POJ23Spa{6(;`l9rfrm+Y#5tey`}a7n zxj}r&-6-rlW)Mmb>%iW@K#I252^@sf@ni*I=Y#NSdA;3KYFLG3sDK!B9c=x0A3;5M z`Nmn`~CIKa(6uflYAThM^1)qrs@JvdxL4YluyQ!+G zvN{RK{)%#PdlA@xEd?=)k8(2Zq&GB2#Lpg^}e&$#k`@m z_l5PQLqlp2RNIU{NC3q26hJBoZ0pyV3!E0#BMJmZWGrmMyb6wl3uL=096>9UWNziq7esAl6w~{&+%lC zlda#vRp@8^MHG(!S>CvBh6s;p42xWOn>pmO6r&jMA2M)jp5WmR%OTQrY}`*a^e-iV zfS5T0c4gT>)tZk}bpc(f*1z237mpt;l@_5SlJZrblBF0V5$uzPYoa!pnM0f+#IKsB z-r2!Dq7q}vUYrxgqEFI*K>8n_%9CRF+rJ(jq(sckUSZAo2F-6uy{;caL^KND7@Q|W z%*Subb3CPIikIkN?4c|cCiX#(`l8Q_=H#17hsusj_z5ybx^70PNo!HCEdY1xiE67g zJt!u=Z=;_=8ENv7LNtG>+{ID-zEmU?{anH;^YwWb)i;cI^6b)+3lx&5 zBwQS&5+-bX3*o3Q@;y>)$5>8J;13Lvz-r>*-a{NO<0mkYzykup_ha-CL(E8$inwr| z?51D}2A`6~kd-^}Oj$<0qWS|23iv<@K}Z^XpsR?usO$Oofe6_7SlFL?nwwta?dW+! zMf6gUZKj@SB&%nl1Q^7u?WcDM`!2mc;F=hs(HdnV zu5NC;ieBlcP^Iq~#JijE->AwfbzounDjhooJxH$+>@0UN&>2gQXUK|e5V<;xE%0%;TR1H z2<)Y260NUdE`|rUM-r5LJ_zp;Z=*^A6LD3%%AVpQmI`%TsFeoFEB)*WidO8kD?T&S zp580?WIka7z{Cfe>!A;If4ClnuP`^)`y_oKmx>eROD<`p?D*4hR;)&uN-|-cRt5Wh*&5QnslSmKtp*9wckc5FL1oC zH8l(6f9z3-ij$&cFWGuo?<=?BncyULQ;{6;2!V&UpC}^L=1i4`@d!B5!bfG41AYsi zC2%*6Hu(a+`5CWwM7P>@h@@XB$E=BGEN0Jnufmp>-+wilGrhVYzqMW2dE7|R*jkxU zX0=GmwpzsbL6&n7<+s3TtvsBN$RW0O;B)Wb=XDOZ>~=g60r2i?=}~R*zLMAW|Fhiy z4@M8E;dQjO{kdD9%-}xl**SVrCI(2^9tYH4>a`)jB&h2iRtvWtgT=dckV4b}yPOJd zlx%ID@5jc{=C{N>#a=1u3hlkcBV_JxNIM^gKK?dNr7EXSes@QNF_iWu#IOyjzGW$F+>t=70&rS872fPGoETiU`u`ZhJ=K@{uvy7m6p}H z*v2MaK(8e8E7M}VM3%9mjW0ml{4L;+B<)M&oKb}V5P%`|-bi@_TrOAR-&FJ3B7@lk zEVkPbdEnr?RzTNCe)AMj1}9TK00kXyIUa2Z6UPrdiIsZxvGV4&b3*LzQo>xvo=Ib#U7mXwCcjoM|wU952gCi z`W$-l``p3#MP!%`R5e=u&#;FF#-e@~E^Y72Wfr;NkW3O;h|ABp*#kKm=5D+iQ^$x6 z88@kkm2`dwZEZ3?Y@`xO!84!Xa{BMwDaiyY*AopvpPw7;*ZP9HuR9uDG@L?v@jR_O zl**)>&AH2*2 z0HB}#%>~HHU~4IA3G59=q=9q%o?^g&@Zp!d{=fF`#yc}SV&zYWv)|{ate2#ko0-5D zaCXF(RBJBk;rL}?J+{PDt{Mt!@QxTQ&0luUHXf033VUDv!B;T*M3pFXc8k!i>fFEo zx?OdK3rrEkgcD4FJ03W{cY)$1_KxF>eK!9){R99ep&wW1ch!gP3pox4^|815sLc*k zViJsBLa;$_N<6NDSBwLf{mTcujVN{_v$8Y*vV#9T14GjDdE-DQrp(td7 zGeW^h8jrO1C)`K3g+3P>$6xq0fByW%xTE@rcD}h$H4avKHN6{ygqkmn#)~Kd!l^!2 zy|}J2aD+G&f%1gCb;QlB*H`pfQ0!P^=)w`j@KvDLP~Hi4J3olhNe3!ADAnm+_k#km zi*CJOPlAF$_ssmDpm_0}x8G-w)v_DpXr|*#OV_7!SsNzkR)EKK`O|v($fkxQ z$%zLBoNr95(kQaVKiJuN@cTFm-ytxQZspEZBh_5@6Gl049OYKD&5+RM4L6k+QgQ|N%qG506AA}(mtDf<$4d<1UcZmbe0K^`;G40l6&L*uo2_Cfd98l?Q3f4CeUcYNZZ4%F{(NSxm7 zKPlb&vOXWIqm`CC$8Pm5{Y{rYoQyZi^gT^z@W6WYdq&&8_W=NbAIl(?P5W{C^&+lXB*)@5CCt_azSA*&nYIHy$<<)a#xtzKz&L_RsLwJhX z6zB3yhuL4!gc3Hui?{%6q*Sz8h)m7Z2c|OCuLFWDf2gD^WfWx_6fae6PB>dzM&AmH z1=6Fb_)qLbn^7u@1U6tPM)3-yE$|MD#(f(J9cm-5D9e6RAx$5%{tD`UH{*^vywuw9 zl(rolu_PCE6LQD~?>#t|p`wkrqWhAD* zp{{dc!AI0e^9+@VS%JTug^II0g2RMr7EYo`F&<~uydx}()alFBT!QO*>ayP!C#lj_ zUd8Y4{6|t+AlGMcywk$&^OT=hOEqREeb;Po7?voU>H}6J@}llFL9k5qh8niW2z2zMqU!mwlMNhZ2(TSqF-p zcab-02j6l7nriN5LjUwVT)=^WQ>FR%?pQyg7W7y2Ili(kD#Mdq)Z)Fy+cK}5BEMUT zlVOSACDnw?hlW0=!!r%m$+RN-17`V0k41LxLVBxJ%B)`Z!s){ucUM%6P~s?D{0Zu$ zL;RHVWq#}Z?1tsK5?d#GYl8-eB&2U^8aYAng#Ui0A~Y};5Do`Co)f4rrg?{58zn{i zBx`O6x4&|Hy!(#c0z8{xVX93QqHFPOQ!TdL3loR$XwpXeN^vrn?#eN;_8a;lNK#I) z$Q%38btEvP0u^8Pc`fMA-~l_wX96z#k#dv6JEy@Xu_-L(iV)vniIMcKsLGYcA*J8T zXQJ2Apb>;LmNUV{-RqW62OWLd>;jW}GK!;(t&eTE47136)R|HqJT*oyfBM~s_WvR{ zN6%hzH};b}h9|HtPPYvZh6MHm?%6Zz90MKrFjhoV9*~Cw? zyvHBUNGy;VwX3fWW#XJnT`vO%yY4U|!mC&t^c0ajf5dM|42=bha@A=KzWf zIt~c&d!@ub9!$fzzGLD{=euBe1_DEgD9wgzq(Y>Qc2E9vQmW zVyekHadtPl27@g%wrd8o<(uPLO+4I}8%rhUIN3Ce5|ZGyfx;{!^juC27w+ghP8{36q-xi{TTO~KDjmW)r`Dmt za&G%&wY7JbL74ImOA$fDKYn{5y-RzGfnU^5N5lUlYMFdp7Nf;AT6J~F!$7;$W9mY* z|8$~_3o`#5BMY}6A$aFPtgTz%HYY`hw@ZlGJPk9+YsE;Xot|!MB$*r)dVAd(nzJxo z@8{_34&fI8_F%aD3_(BE=Y7Vg;NsTbRSESR zrq|CF{ers83@TjeU4|?!@7To#8NDF!nx_zW$R}YjrAkBc!)ny<4!TRTcfD{=^uluK z>t38bG!GD${WW#@mPqN~QE^z2=fG5Hel|G!7kewercGjve1oB&iOXjaTJd5GXtPSj z2kqA~T^tP7Ug~F!@VvonrD=MwSfXkMo8{RJ36w|7<#K;6i3v52w(%#?dd=m&Xrvkg zjwI2|n;ITeSQ=lFF-w}Ow>Vjj3kT=;o@j$ZpVOJqlc2Fsp%2Ukzf-#r{(mCF7EQ1B z)mj1+QsoD?$-h5(xGg>+AgBWKrb9@+`adFG>VNGfE`B!T6D*qJ0z?QG?y&DEoKuogU3k? z6S5HD5yqu<_cbal7Ntr0G0BXwNj#$_$aEuN((6PHBlSF9)_TF`G-3PIeDd>tYrJP@SzF?0WaOKnGYZoUAKH z!;H`_Y9o-hgF;-b|u*Z)uPuf*F)-n?h5 ZpMtBThx_qz@U&onjHIGOm6-9T{{==`;wb Date: Fri, 21 Dec 2018 10:15:24 -0500 Subject: [PATCH 009/157] text changes --- src/main/java/Bundle.properties | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/Bundle.properties b/src/main/java/Bundle.properties index d857d6d8026..7035b657e91 100644 --- a/src/main/java/Bundle.properties +++ b/src/main/java/Bundle.properties @@ -115,7 +115,7 @@ contact.header=Contact {0} contact.dataverse.header=Email Dataverse Contact contact.dataset.header=Email Dataset Contact contact.to=To -contact.support=Support +contact.support=TDL Dataverse Support contact.from=From contact.from.required=User email is required. contact.from.invalid=Email is invalid. @@ -260,10 +260,10 @@ login.System=Login System login.forgot.text=Forgot your password? login.builtin=Dataverse Account login.institution=Institutional Account -login.institution.blurb=Log in or sign up with your institutional account — learn more. +login.institution.blurb=Log in or sign up with your institutional account — learn more. If you are not affiliated with a TDR member institution (see dropdown menu), please use the Google Login option. login.institution.support.beforeLink=Leaving your institution? Please contact login.institution.support.afterLink=for assistance. -login.builtin.credential.usernameOrEmail=Username/Email +login.builtin.credential.usernameOrEmail=Admin ID login.builtin.credential.password=Password login.builtin.invalidUsernameEmailOrPassword=The username, email address, or password you entered is invalid. Need assistance accessing your account? login.echo.credential.name=Name @@ -274,16 +274,16 @@ login.error=Error validating the username, email address, or password. Please tr user.error.cannotChangePassword=Sorry, your password cannot be changed. Please contact your system administrator. user.error.wrongPassword=Sorry, wrong password. login.button=Log In with {0} -login.button.orcid=Create or Connect your ORCID +login.button.orcid=Google Login - No TDR affiliation # authentication providers auth.providers.title=Other options auth.providers.tip=You can convert a Dataverse account to use one of the options above. Learn more. -auth.providers.title.builtin=Username/Email +auth.providers.title.builtin=Admin ID auth.providers.title.shib=Your Institution auth.providers.title.orcid=ORCID auth.providers.title.google=Google auth.providers.title.github=GitHub -auth.providers.blurb=Log in or sign up with your {0} account — learn more. Having trouble? Please contact {3} for assistance. +auth.providers.blurb=Log in or sign up with your Google account — learn more. If you are not affiliated with a TDR member institution, please use the Google Login option. Having trouble? Please contact {3} for assistance. auth.providers.persistentUserIdName.orcid=ORCID iD auth.providers.persistentUserIdName.github=ID auth.providers.persistentUserIdTooltip.orcid=ORCID provides a persistent digital identifier that distinguishes you from other researchers. @@ -322,7 +322,7 @@ shib.welcomeExistingUserMessageDefaultInstitution=your institution shib.dataverseUsername=Dataverse Username shib.currentDataversePassword=Current Dataverse Password shib.accountInformation=Account Information -shib.offerToCreateNewAccount=This information is provided by your institution and will be used to create your Dataverse account. +shib.offerToCreateNewAccount=Contact your TDR liaison to get help and training. Published content cannot be easily deleted. shib.passwordRejected=Validation Error - Your account can only be converted if you provide the correct password for your existing account. # oauth2/firstLogin.xhtml From 799892906c8256a96c247bfa89c46db9df0dfc38 Mon Sep 17 00:00:00 2001 From: Jim Myers Date: Wed, 26 Dec 2018 15:27:50 -0500 Subject: [PATCH 010/157] TDRDV-46 --- src/main/java/edu/harvard/iq/dataverse/DataversePage.java | 3 +++ src/main/webapp/dataverse.xhtml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 0bd67f0f24e..f95908df058 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -46,6 +46,8 @@ import javax.faces.model.SelectItem; import org.apache.commons.lang.StringEscapeUtils; import org.apache.commons.lang.StringUtils; +import org.primefaces.PrimeFaces; +import org.primefaces.context.RequestContext; import org.primefaces.event.TransferEvent; /** @@ -506,6 +508,7 @@ public void updateInclude(Long mdbId, long dsftId) { } } } + PrimeFaces.current().scrollTo("dsft"+dsftId); } public List resetSelectItems(DatasetFieldType typeIn) { diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index b287d151122..33f1a378cca 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -264,7 +264,7 @@ - +
Date: Thu, 27 Dec 2018 17:02:59 -0500 Subject: [PATCH 011/157] TDRDV-46 use javascript to scroll within panel after update --- .../java/edu/harvard/iq/dataverse/DataversePage.java | 2 +- src/main/webapp/dataverse.xhtml | 11 +++++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index f95908df058..e7493e936c5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -508,7 +508,7 @@ public void updateInclude(Long mdbId, long dsftId) { } } } - PrimeFaces.current().scrollTo("dsft"+dsftId); + PrimeFaces.current().executeScript("scrollAfterUpdate();"); } public List resetSelectItems(DatasetFieldType typeIn) { diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 33f1a378cca..16268969756 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -264,10 +264,10 @@ - +
@@ -743,6 +743,9 @@ diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index f26218913f3..4b3a9c5ee81 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -1,3 +1,40 @@ +var fileList = []; +var observer2=null; + +function setupDirectUpload() { + +var fileInput=document.getElementById('datasetForm:fileUpload_input'); +fileInput.addEventListener('change', function(event) { +fileList=[]; +for(var i=0;i Date: Fri, 6 Dec 2019 10:28:12 -0500 Subject: [PATCH 054/157] toggle auto upload based on directupload property --- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 5 +++++ src/main/webapp/editFilesFragment.xhtml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 44dab4fc3cd..e50f566e028 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -9,6 +9,7 @@ import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; import edu.harvard.iq.dataverse.datasetutility.FileReplaceException; import edu.harvard.iq.dataverse.datasetutility.FileReplacePageHelper; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; @@ -350,6 +351,10 @@ public boolean doesSessionUserHaveDataSetPermission(Permission permissionToCheck return hasPermission; } + public boolean directUploadEnabled() { + return Boolean.getBoolean(System.getProperty("dataverse.file." + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + ".upload-redirect", "false")); + } + public void reset() { // ? } diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 315abe117ce..030726cff2d 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -124,7 +124,7 @@ Date: Mon, 9 Dec 2019 13:23:50 -0500 Subject: [PATCH 055/157] implement proxying --- .../harvard/iq/dataverse/api/Datasets.java | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index c88ed4ca981..ce039521e8a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1450,21 +1450,10 @@ public Response getUploadUrl(@PathParam("id") String idSupplied) { return error(Response.Status.FORBIDDEN, "You are not permitted to upload files to this dataset."); } - String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); - boolean directEnabled = Boolean.getBoolean(System.getProperty("dataverse.files." + driverId + ".upload-redirect", "false")); - if(!directEnabled) { - return error(Response.Status.NOT_FOUND, "Direct upload not supported for files in this dataset."); - } - String bucket = System.getProperty("dataverse.files." + driverId + ".bucket-name") + "/"; - String sid = bucket+ dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + FileUtil.generateStorageIdentifier(); - S3AccessIO s3io = new S3AccessIO(sid, driverId); - String url = null; - try { - url = s3io.generateTemporaryS3UploadUrl(); - } catch (IOException e) { - logger.warning("Identifier Collision"); - e.printStackTrace(); - } + String url = FileUtil.getDirectUploadUrl(dataset); + if(url==null) { + return error(Response.Status.NOT_FOUND,"Direct upload not supported for files in this dataset: " + dataset.getId()); + } return ok(url); } catch (WrappedResponse wr) { return wr.getResponse(); From 9c01f28d997748dc9d577959aa4c76488541ae7a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 9 Dec 2019 13:25:25 -0500 Subject: [PATCH 056/157] implement proxy --- .../harvard/iq/dataverse/util/FileUtil.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index cbad62329e8..ade2040b44c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -24,11 +24,13 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.DataFile.ChecksumType; import edu.harvard.iq.dataverse.DataFileServiceBean; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.DatasetVersion; import edu.harvard.iq.dataverse.FileMetadata; import edu.harvard.iq.dataverse.TermsOfUseAndAccess; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; +import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; import edu.harvard.iq.dataverse.datasetutility.FileExceedsMaxSizeException; import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable; @@ -1666,5 +1668,33 @@ public static DatasetThumbnail getThumbnail(DataFile file) { public static boolean isPackageFile(DataFile dataFile) { return DataFileServiceBean.MIME_TYPE_PACKAGE_FILE.equalsIgnoreCase(dataFile.getContentType()); } + + public static String getDirectUploadUrl(Dataset dataset) { + String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); + boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect"); + //Should only be requested when it is allowed, but we'll log a warning otherwise + if(!directEnabled) { + logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId()); + return null; + } + String bucket = System.getProperty("dataverse.files." + driverId + ".bucket-name") + "/"; + String sid = bucket+ dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + FileUtil.generateStorageIdentifier(); + S3AccessIO s3io = new S3AccessIO(sid, driverId); + String url = null; + try { + url = s3io.generateTemporaryS3UploadUrl(); + } catch (IOException e) { + logger.warning("Identifier Collision"); + e.printStackTrace(); + } + String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); + + String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url" + + ""); + if(proxy!=null) { + url.replace(endpoint, proxy); + } + return url; + } } From 0fe9918afa18ab4cd39e7d4822546d27d48a9100 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 9 Dec 2019 13:27:54 -0500 Subject: [PATCH 057/157] further updates to direct upload files --- .../iq/dataverse/EditDatafilesPage.java | 10 +- .../iq/dataverse/dataaccess/DataAccess.java | 7 ++ src/main/webapp/editFilesFragment.xhtml | 6 +- src/main/webapp/resources/js/fileupload.js | 110 +++++++++++++----- 4 files changed, 101 insertions(+), 32 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index e50f566e028..9638cda4ba8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -11,6 +11,7 @@ import edu.harvard.iq.dataverse.datasetutility.FileReplacePageHelper; import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; +import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; @@ -84,6 +85,8 @@ import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; +import javax.ws.rs.core.Response; + import org.apache.commons.lang.StringUtils; import org.primefaces.PrimeFaces; //import org.primefaces.context.RequestContext; @@ -352,7 +355,9 @@ public boolean doesSessionUserHaveDataSetPermission(Permission permissionToCheck } public boolean directUploadEnabled() { - return Boolean.getBoolean(System.getProperty("dataverse.file." + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + ".upload-redirect", "false")); + logger.info(this.dataset.getDataverseContext().getAlias() + " " + this.dataset.getDataverseContext().getAffiliation() + " " + DataAccess.getStorageDriverId(this.dataset.getDataverseContext())); + logger.info("denabled: " + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + " " + Boolean.getBoolean("dataverse.files." + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + ".upload-redirect")); + return Boolean.getBoolean("dataverse.files." + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + ".upload-redirect"); } public void reset() { @@ -1722,6 +1727,9 @@ public String getRsyncScriptFilename() { return rsyncScriptFilename; } + public void requestDirectUploadUrl() { + PrimeFaces.current().executeScript("uploadFileDirectly('" + FileUtil.getDirectUploadUrl(dataset) + "')"); + } public void uploadFinished() { // This method is triggered from the page, by the @@ -124,7 +125,7 @@ + diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 4b3a9c5ee81..febf7c59a57 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -1,39 +1,91 @@ var fileList = []; var observer2=null; +var datasetId=null; -function setupDirectUpload() { - -var fileInput=document.getElementById('datasetForm:fileUpload_input'); -fileInput.addEventListener('change', function(event) { -fileList=[]; -for(var i=0;i')); + $.ajax({ + url: url, + type: 'POST', + data: data, + cache: false, + processData: false, + success: reportUpload, + xhr: function() { + var myXhr = $.ajaxSettings.xhr(); + if(myXhr.upload) { + myXhr.upload.addEventListener('progress', function(e) { + if(e.lengthComputable) { + $('progress').attr({ + value:e.loaded, + max:e.total + }) + } + }) + } + } + }); + //perform post - check cors issues + //trigger gui swap/server file add + //handle cancel buttons? + } - } - }); - }; - if(observer2 !=null) { - observer2.disconnect(); - } - observer2 = new MutationObserver(callback); - observer2.observe(fileInput.parentElement,config); +function reportUpload(data){ + console.log('Done ' + data ); } function removeErrors() { From 518e2d27a95a0e943f75b2739a8ed4e0f9723e60 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 13 Dec 2019 17:43:01 -0500 Subject: [PATCH 058/157] add driver to upload url response --- src/main/java/edu/harvard/iq/dataverse/api/Datasets.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index ce039521e8a..bdde4aa21c9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1454,7 +1454,9 @@ public Response getUploadUrl(@PathParam("id") String idSupplied) { if(url==null) { return error(Response.Status.NOT_FOUND,"Direct upload not supported for files in this dataset: " + dataset.getId()); } - return ok(url); + String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); + String response = "{\"url\":\"" + url + "\",\"driverId\":\"" + driverId + "\"}"; + return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); } From 076c75a3335c7543d1459bed3dafd08c647fe899 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 13 Dec 2019 17:43:34 -0500 Subject: [PATCH 059/157] fix proxy url replacement --- .../java/edu/harvard/iq/dataverse/util/FileUtil.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index ade2040b44c..1a799ebd786 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1688,12 +1688,13 @@ public static String getDirectUploadUrl(Dataset dataset) { e.printStackTrace(); } String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); - - String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url" - + ""); + logger.info("endpoint: " + endpoint); + String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url"); + logger.info("proxy: " + proxy); if(proxy!=null) { - url.replace(endpoint, proxy); + url = url.replace(endpoint, proxy); } + logger.info("url: " + url); return url; } From 2f7102d3e1046baff5f99efc08b52a5d3c3d6b65 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 10:01:16 -0500 Subject: [PATCH 060/157] single file, no hash upload via GUI minimally working --- .../iq/dataverse/EditDatafilesPage.java | 174 +++++++++++++++++- .../harvard/iq/dataverse/api/Datasets.java | 21 ++- .../iq/dataverse/dataaccess/DataAccess.java | 18 +- .../iq/dataverse/dataaccess/S3AccessIO.java | 80 ++++---- .../datasetutility/AddReplaceFileHelper.java | 3 +- .../datasetutility/FileReplacePageHelper.java | 7 +- .../dataverse/ingest/IngestServiceBean.java | 14 +- .../harvard/iq/dataverse/util/FileUtil.java | 62 ++++--- src/main/webapp/editFilesFragment.xhtml | 1 + src/main/webapp/resources/js/fileupload.js | 54 ++++-- 10 files changed, 329 insertions(+), 105 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 9638cda4ba8..d26d0a33112 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean; import edu.harvard.iq.dataverse.api.AbstractApiBean; +import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -10,8 +11,10 @@ import edu.harvard.iq.dataverse.datasetutility.FileReplaceException; import edu.harvard.iq.dataverse.datasetutility.FileReplacePageHelper; import edu.harvard.iq.dataverse.dataaccess.DataAccess; +import edu.harvard.iq.dataverse.dataaccess.DataAccessOption; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; +import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.datacapturemodule.DataCaptureModuleUtil; import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; @@ -1728,7 +1731,21 @@ public String getRsyncScriptFilename() { } public void requestDirectUploadUrl() { - PrimeFaces.current().executeScript("uploadFileDirectly('" + FileUtil.getDirectUploadUrl(dataset) + "')"); + S3AccessIO s3io = FileUtil.getS3AccessForDirectUpload(dataset); + if(s3io == null) { + FacesContext.getCurrentInstance().addMessage(uploadComponentId, new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.uploadWarning"), "Direct upload not supported for this dataset")); + } + String url = null; + String storageIdentifier = null; + try { + url = s3io.generateTemporaryS3UploadUrl(); + storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation()); + } catch (IOException io) { + logger.warning(io.getMessage()); + FacesContext.getCurrentInstance().addMessage(uploadComponentId, new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.uploadWarning"), "Issue in connecting to S3 store for direct upload")); + } + + PrimeFaces.current().executeScript("uploadFileDirectly('" + url + "','" + storageIdentifier + "')"); } public void uploadFinished() { @@ -1758,6 +1775,7 @@ public void uploadFinished() { for (DataFile dataFile : uploadedFiles) { fileMetadatas.add(dataFile.getFileMetadata()); newFiles.add(dataFile); + logger.info("Added " + dataFile.getFileMetadata().getLabel()); } @@ -1821,7 +1839,7 @@ private void handleReplaceFileUpload(FacesEvent event, InputStream inputStream, uploadComponentId = event.getComponent().getClientId(); - if (fileReplacePageHelper.handleNativeFileUpload(inputStream, + if (fileReplacePageHelper.handleNativeFileUpload(inputStream,null, fileName, contentType )){ @@ -1879,6 +1897,32 @@ the uploadFinished() method, triggered next, after the upload event // new FacesMessage(FacesMessage.SEVERITY_ERROR, "upload failure", uploadWarningMessage)); } } + + private void handleReplaceFileUpload(String fullStorageLocation, + String fileName, + String contentType){ + + fileReplacePageHelper.resetReplaceFileHelper(); + + saveEnabled = false; + + if (fileReplacePageHelper.handleNativeFileUpload(null, fullStorageLocation, + fileName, + contentType + )){ + saveEnabled = true; + + /** + * If the file content type changed, let the user know + */ + if (fileReplacePageHelper.hasContentTypeWarning()){ + //Add warning to popup instead of page for Content Type Difference + setWarningMessageForPopUp(fileReplacePageHelper.getContentTypeWarning()); + } + } else { + uploadWarningMessage = fileReplacePageHelper.getErrorMessages(); + } + } private String uploadWarningMessage = null; private String uploadSuccessMessage = null; @@ -1967,6 +2011,129 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { } } + /** + * Using information from the DropBox choose, ingest the chosen files + * https://www.dropbox.com/developers/dropins/chooser/js + * + * @param event + */ + public void handleExternalUpload() { + Map paramMap = FacesContext.getCurrentInstance().getExternalContext().getRequestParameterMap(); + + this.uploadComponentId = paramMap.get("uploadComponentId"); + String fullStorageIdentifier = paramMap.get("fullStorageIdentifier"); + String fileName = paramMap.get("fileName"); + String contentType = paramMap.get("contentType"); + String checksumType = paramMap.get("checksumType"); + String checksumValue = paramMap.get("checksumValue"); + + logger.info("UCI" + uploadComponentId); + logger.info("FSI" + fullStorageIdentifier); + logger.info("FN" + fileName); + logger.info("CT" + contentType); + logger.info("CST" + checksumType); + logger.info("CDV" + checksumValue); + int lastColon = fullStorageIdentifier.lastIndexOf(':'); + String storageLocation= fullStorageIdentifier.substring(0,lastColon) + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + fullStorageIdentifier.substring(lastColon+1); + logger.info("SL " + storageLocation); + if (!uploadInProgress) { + uploadInProgress = true; + } + logger.fine("handleExternalUpload"); + + StorageIO sio; + String localWarningMessage = null; + try { + sio = DataAccess.getDirectStorageIO(storageLocation); + + //Populate metadata + sio.open(DataAccessOption.READ_ACCESS); + //get file size + long fileSize = sio.getSize(); + logger.info("FS " + fileSize); + + /* ---------------------------- + Check file size + - Max size NOT specified in db: default is unlimited + - Max size specified in db: check too make sure file is within limits + // ---------------------------- */ + if ((!this.isUnlimitedUploadFileSize()) && (fileSize > this.getMaxFileUploadSizeInBytes())) { + String warningMessage = "Uploaded file \"" + fileName + "\" exceeded the limit of " + fileSize + " bytes and was not uploaded."; + sio.delete(); + //msg(warningMessage); + //FacesContext.getCurrentInstance().addMessage(event.getComponent().getClientId(), new FacesMessage(FacesMessage.SEVERITY_ERROR, "upload failure", warningMessage)); + if (localWarningMessage == null) { + localWarningMessage = warningMessage; + } else { + localWarningMessage = localWarningMessage.concat("; " + warningMessage); + } + } else { + // ----------------------------------------------------------- + // Is this a FileReplaceOperation? If so, then diverge! + // ----------------------------------------------------------- + if (this.isFileReplaceOperation()){ + this.handleReplaceFileUpload(storageLocation, fileName, FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT); + this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0)); + return; + } + // ----------------------------------------------------------- + List datafiles = new ArrayList<>(); + + // ----------------------------------------------------------- + // Send it through the ingest service + // ----------------------------------------------------------- + try { + + // Note: A single uploaded file may produce multiple datafiles - + // for example, multiple files can be extracted from an uncompressed + // zip file. + //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream"); + if(contentType==null) { + contentType = "application/octet-stream"; + } + if(DataFile.ChecksumType.fromString(checksumType) != DataFile.ChecksumType.MD5 ) { + String warningMessage = "Non-MD5 checksums not yet supported in external uploads"; + if (localWarningMessage == null) { + localWarningMessage = warningMessage; + } else { + localWarningMessage = localWarningMessage.concat("; " + warningMessage); + } + + } + datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, storageLocation, checksumValue, systemConfig); + logger.info("Created " + datafiles.size() + " files."); + } catch (IOException ex) { + logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName}); + } + + if (datafiles == null){ + logger.log(Level.SEVERE, "Failed to create DataFile for file {0}", new Object[]{fileName}); + }else{ + // ----------------------------------------------------------- + // Check if there are duplicate files or ingest warnings + // ----------------------------------------------------------- + uploadWarningMessage = processUploadedFileList(datafiles); + logger.info("Warning message during upload: " + uploadWarningMessage); + } + if(!uploadInProgress) { + logger.warning("Upload in progress cancelled"); + for (DataFile newFile : datafiles) { + deleteTempFile(newFile); + } + } + } + } catch (IOException e) { + logger.log(Level.SEVERE, "Failed to create DataFile for file {0}: {1}", new Object[]{fileName, e.getMessage()}); + } + if (localWarningMessage != null) { + if (uploadWarningMessage == null) { + uploadWarningMessage = localWarningMessage; + } else { + uploadWarningMessage = localWarningMessage.concat("; " + uploadWarningMessage); + } + } + } + /** * After uploading via the site or Dropbox, * check the list of DataFile objects @@ -1980,7 +2147,7 @@ public void handleFileUpload(FileUploadEvent event) throws IOException { private boolean uploadInProgress = false; private String processUploadedFileList(List dFileList) { - + logger.info("Processing list of " + dFileList.size() + " files."); if (dFileList == null) { return null; } @@ -2042,6 +2209,7 @@ private String processUploadedFileList(List dFileList) { dataFile.setPreviewImageAvailable(true); } uploadedFiles.add(dataFile); + logger.info("Added file to list: " + dataFile.getFileMetadata().getLabel()); // We are NOT adding the fileMetadata to the list that is being used // to render the page; we'll do that once we know that all the individual uploads // in this batch (as in, a bunch of drag-and-dropped files) have finished. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index bdde4aa21c9..322c9922a3f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1449,13 +1449,24 @@ public Response getUploadUrl(@PathParam("id") String idSupplied) { if (!canUpdateDataset) { return error(Response.Status.FORBIDDEN, "You are not permitted to upload files to this dataset."); } - - String url = FileUtil.getDirectUploadUrl(dataset); - if(url==null) { + S3AccessIO s3io = FileUtil.getS3AccessForDirectUpload(dataset); + if(s3io == null) { return error(Response.Status.NOT_FOUND,"Direct upload not supported for files in this dataset: " + dataset.getId()); } - String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); - String response = "{\"url\":\"" + url + "\",\"driverId\":\"" + driverId + "\"}"; + String url = null; + String storageIdentifier = null; + try { + url = s3io.generateTemporaryS3UploadUrl(); + storageIdentifier = FileUtil.getStorageIdentifierFromLocation(s3io.getStorageLocation()); + } catch (IOException io) { + logger.warning(io.getMessage()); + throw new WrappedResponse(io, error( Response.Status.INTERNAL_SERVER_ERROR, "Could not create process direct upload request")); + } + + + JsonObjectBuilder response = Json.createObjectBuilder() + .add("url", url) + .add("storageIdentifier", storageIdentifier ); return ok(response); } catch (WrappedResponse wr) { return wr.getResponse(); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index db24fa94382..9c77cdf6293 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -91,24 +91,24 @@ public static StorageIO getStorageIO(T dvObject, DataAcc // Experimental extension of the StorageIO system allowing direct access to // stored physical files that may not be associated with any DvObjects - public static StorageIO getDirectStorageIO(String storageLocation) throws IOException { - String[] response = getDriverIdAndStorageId(storageLocation); + public static StorageIO getDirectStorageIO(String fullStorageLocation) throws IOException { + String[] response = getDriverIdAndStorageLocation(fullStorageLocation); String storageDriverId = response[0]; - String storageIdentifier=response[1]; + String storageLocation=response[1]; String storageType = getDriverType(storageDriverId); switch(storageType) { case "file": - return new FileAccessIO<>(storageIdentifier, storageDriverId); + return new FileAccessIO<>(storageLocation, storageDriverId); case "s3": - return new S3AccessIO<>(storageIdentifier, storageDriverId); + return new S3AccessIO<>(storageLocation, storageDriverId); case "swift": - return new SwiftAccessIO<>(storageIdentifier, storageDriverId); + return new SwiftAccessIO<>(storageLocation, storageDriverId); default: throw new IOException("getDirectStorageIO: Unsupported storage method."); } } - public static String[] getDriverIdAndStorageId(String storageLocation) { + public static String[] getDriverIdAndStorageLocation(String storageLocation) { //default if no prefix String storageIdentifier=storageLocation; int separatorIndex = storageLocation.indexOf("://"); @@ -120,6 +120,10 @@ public static String[] getDriverIdAndStorageId(String storageLocation) { return new String[]{storageDriverId, storageIdentifier}; } + public static String getStorarageIdFromLocation(String location) { + return location.substring(location.lastIndexOf('/')+1); + } + public static String getDriverType(String driverId) { return System.getProperty("dataverse.files." + driverId + ".type", "file"); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index d0949e821c8..13c8a8a1e17 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -1,6 +1,7 @@ package edu.harvard.iq.dataverse.dataaccess; import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; import com.amazonaws.HttpMethod; import com.amazonaws.SdkClientException; import com.amazonaws.auth.profile.ProfileCredentialsProvider; @@ -101,11 +102,12 @@ public S3AccessIO(T dvObject, DataAccessRequest req, String driverId) { } - public S3AccessIO(String storageLocation, String driverId) { + public S3AccessIO(String storageLocation, String driverId) throws IOException { this(null, null, driverId); // TODO: validate the storage location supplied bucketName = storageLocation.substring(0,storageLocation.indexOf('/')); key = storageLocation.substring(storageLocation.indexOf('/')+1); + } public S3AccessIO(T dvObject, DataAccessRequest req, @NotNull AmazonS3 s3client, String driverId) { @@ -126,7 +128,7 @@ public S3AccessIO(T dvObject, DataAccessRequest req, @NotNull AmazonS3 s3client, private String s3profile = "default"; private String bucketName = null; - private String key; + private String key = null; @Override public void open(DataAccessOption... options) throws IOException { @@ -212,7 +214,18 @@ public void open(DataAccessOption... options) throws IOException { } else if (dvObject instanceof Dataverse) { throw new IOException("Data Access: Storage driver does not support dvObject type Dataverse yet"); } else { + //Direct access, e.g. for external upload - no associated DVobject yet, but we want to be abel to get the size + if(key!=null) { + ObjectMetadata objectMetadata = null; + try { + objectMetadata = s3.getObjectMetadata(bucketName, key); + } catch (SdkClientException sce) { + throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); + } + this.setSize(objectMetadata.getContentLength()); + }else { throw new IOException("Data Access: Invalid DvObject type"); + } } } @@ -773,7 +786,7 @@ public String generateTemporaryS3Url() throws IOException { key = getMainFileKey(); java.util.Date expiration = new java.util.Date(); long msec = expiration.getTime(); - msec += 1000 * getUrlExpirationMinutes(); + msec += 60 * 1000 * getUrlExpirationMinutes(); expiration.setTime(msec); GeneratePresignedUrlRequest generatePresignedUrlRequest = @@ -821,40 +834,37 @@ public String generateTemporaryS3Url() throws IOException { } public String generateTemporaryS3UploadUrl() throws IOException { - //Questions: - // Q. how long should the download url work? - // A. 1 hour by default seems like an OK number. Making it configurable seems like a good idea too. -- L.A. - if (s3 == null) { - throw new IOException("ERROR: s3 not initialised. "); - } - //ToDO: Check for existing file given storage ID (could check bucket and/or DV Datafile - if(true) { - key = getMainFileKey(); - java.util.Date expiration = new java.util.Date(); - long msec = expiration.getTime(); - msec += 1000 * getUrlExpirationMinutes(); - expiration.setTime(msec); - - GeneratePresignedUrlRequest generatePresignedUrlRequest = - new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); - URL s; - try { - s = s3.generatePresignedUrl(generatePresignedUrlRequest); - } catch (SdkClientException sce) { - //throw new IOException("SdkClientException generating temporary S3 url for "+key+" ("+sce.getMessage()+")"); - s = null; - } + key = getMainFileKey(); + java.util.Date expiration = new java.util.Date(); + long msec = expiration.getTime(); + msec += 60 * 1000 * getUrlExpirationMinutes(); + expiration.setTime(msec); - if (s != null) { - return s.toString(); - } - - //throw new IOException("Failed to generate temporary S3 url for "+key); - return null; - } else { - throw new IOException("Data Access: GenerateTemporaryS3UploadUrl: StorageIdentifier already in use!"); - } + GeneratePresignedUrlRequest generatePresignedUrlRequest = + new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); + URL presignedUrl; + try { + presignedUrl = s3.generatePresignedUrl(generatePresignedUrlRequest); + } catch (SdkClientException sce) { + //throw new IOException("SdkClientException generating temporary S3 url for "+key+" ("+sce.getMessage()+")"); + presignedUrl = null; + } + String urlString = null; + if (presignedUrl != null) { + String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); + logger.info("endpoint: " + endpoint); + String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url"); + logger.info("proxy: " + proxy); + if(proxy!=null) { + urlString = presignedUrl.toString().replace(endpoint, proxy); + } else { + urlString = presignedUrl.toString(); + } + } + + //throw new IOException("Failed to generate temporary S3 url for "+key); + return urlString; } int getUrlExpirationMinutes() { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 700b04ab0d9..ea5b5ed3d5e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -442,6 +442,7 @@ public boolean runReplaceFromUI_Phase1(Long oldFileId, String newFileName, String newFileContentType, InputStream newFileInputStream, + String fullStorageId, OptionalFileParams optionalFileParams){ @@ -463,7 +464,7 @@ public boolean runReplaceFromUI_Phase1(Long oldFileId, return this.runAddReplacePhase1(fileToReplace.getOwner(), newFileName, newFileContentType, - null, + fullStorageId, newFileInputStream, optionalFileParams); diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java index e6d7c1e5ebe..94f247e6419 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java @@ -96,14 +96,14 @@ public boolean resetReplaceFileHelper(){ * Handle native file replace * @param event */ - public boolean handleNativeFileUpload(InputStream inputStream, String fileName, String fileContentType) { + public boolean handleNativeFileUpload(InputStream inputStream, String fullStorageId, String fileName, String fileContentType) { phase1Success = false; // Preliminary sanity check // - if (inputStream == null){ - throw new NullPointerException("inputStream cannot be null"); + if ((inputStream == null)&&(fullStorageId==null)){ + throw new NullPointerException("inputStream and storageId cannot both be null"); } if (fileName == null){ throw new NullPointerException("fileName cannot be null"); @@ -118,6 +118,7 @@ public boolean handleNativeFileUpload(InputStream inputStream, String fileName, fileName, fileContentType, inputStream, + fullStorageId, null ); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index c9f0eed779c..83ba0cdb88b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -176,22 +176,22 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List dataAccess = null; try { - logger.fine("Attempting to create a new storageIO object for " + storageId); - dataAccess = DataAccess.createNewStorageIO(dataFile, storageId); + logger.fine("Attempting to create a new storageIO object for " + storageLocation); + dataAccess = DataAccess.createNewStorageIO(dataFile, storageLocation); logger.fine("Successfully created a new storageIO object."); /* @@ -252,7 +252,7 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List generatedTempFiles = listGeneratedTempFiles(Paths.get(FileUtil.getFilesTempDirectory()), - storageId); + storageLocation); if (generatedTempFiles != null) { for (Path generated : generatedTempFiles) { if (savedSuccess) { // no need to try to save this aux file permanently, if we've failed to diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 1a799ebd786..7318f082d04 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -82,6 +82,8 @@ import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable; import org.apache.commons.io.FilenameUtils; +import com.amazonaws.AmazonServiceException; + /** * a 4.0 implementation of the DVN FileUtil; @@ -1669,33 +1671,39 @@ public static boolean isPackageFile(DataFile dataFile) { return DataFileServiceBean.MIME_TYPE_PACKAGE_FILE.equalsIgnoreCase(dataFile.getContentType()); } - public static String getDirectUploadUrl(Dataset dataset) { - String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); - boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect"); - //Should only be requested when it is allowed, but we'll log a warning otherwise - if(!directEnabled) { - logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId()); - return null; - } - String bucket = System.getProperty("dataverse.files." + driverId + ".bucket-name") + "/"; - String sid = bucket+ dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + FileUtil.generateStorageIdentifier(); - S3AccessIO s3io = new S3AccessIO(sid, driverId); - String url = null; - try { - url = s3io.generateTemporaryS3UploadUrl(); - } catch (IOException e) { - logger.warning("Identifier Collision"); - e.printStackTrace(); - } - String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); - logger.info("endpoint: " + endpoint); - String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url"); - logger.info("proxy: " + proxy); - if(proxy!=null) { - url = url.replace(endpoint, proxy); - } - logger.info("url: " + url); - return url; + public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) { + String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); + boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect"); + //Should only be requested when it is allowed, but we'll log a warning otherwise + if(!directEnabled) { + logger.warning("Direct upload not supported for files in this dataset: " + dataset.getId()); + return null; + } + S3AccessIO s3io = null; + String bucket = System.getProperty("dataverse.files." + driverId + ".bucket-name") + "/"; + String sid = null; + int i=0; + while (s3io==null && i<5) { + sid = bucket+ dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + FileUtil.generateStorageIdentifier(); + try { + s3io = new S3AccessIO(sid, driverId); + if(s3io.exists()) { + s3io=null; + i=i+1; + } + + } catch (Exception e) { + i=i+1; + } + + } + return s3io; + } + + public static String getStorageIdentifierFromLocation(String location) { + int driverEnd = location.indexOf("://") + 3; + int bucketEnd = driverEnd + location.substring(driverEnd).indexOf("/"); + return location.substring(0,bucketEnd) + ":" + location.substring(location.lastIndexOf("/") + 1); } } diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index db17829ae4e..900ef90e9ed 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -520,6 +520,7 @@ + diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index febf7c59a57..c6e1b18148b 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -11,7 +11,7 @@ function setupDirectUpload(enabled, theDatasetId) { fileInput.addEventListener('change', function(event) { fileList=[]; for(var i=0;i')); + $('.ui-fileupload-progress').append($('')); $.ajax({ url: url, - type: 'POST', + type: 'PUT', data: data, cache: false, processData: false, - success: reportUpload, + success: function () { + reportUpload(storageId, fileList[0]) + }, + error: function(jqXHR, textStatus, errorThrown) { + console.log('Failure: ' + jqXHR.status); + console.log('Failure: ' + errorThrown); + }, xhr: function() { var myXhr = $.ajaxSettings.xhr(); if(myXhr.upload) { @@ -72,22 +78,34 @@ function uploadFileDirectly(url) { $('progress').attr({ value:e.loaded, max:e.total - }) + }); } - }) + }); } + return myXhr; } }); - //perform post - check cors issues + console.log('after ajax'); +//perform post - check cors issues //trigger gui swap/server file add //handle cancel buttons? - + } -function reportUpload(data){ - console.log('Done ' + data ); +function reportUpload(storageId, file){ + + //storageId is not the location - has a : separator and no path elements from dataset + + //(String uploadComponentId, String fullStorageIdentifier, String fileName, String contentType, String checksumType, String checksumValue) + handleExternalUpload([{name:'uploadComponentId', value:'datasetForm:fileUpload'}, {name:'fullStorageIdentifier', value:storageId}, {name:'fileName', value:file.name}, {name:'contentType', value:file.type}, {name:'checksumType', value:'MD5'}, {name:'checksumValue', value:0}]); + console.log('Done ' + storageId + " " + file.name ); + $('.ui-fileupload-files').html(""); + fileList=[]; + uploadFinished(document.getElementById('datasetForm:fileUpload_input')); + } + function removeErrors() { var errors = document.getElementsByClassName("ui-fileupload-error"); for(i=errors.length-1; i >=0; i--) { @@ -136,6 +154,8 @@ function uploadFinished(fileupload) { observer.disconnect(); observer=null; } + } else { + console.log('Still ' + fileupload.files.length + ' files' ); } } From d8de35fb687d34bb0021465b7dfbae2dd19c5c2c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 12:10:24 -0500 Subject: [PATCH 061/157] send ID without dataset path --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index d26d0a33112..fae88a20c07 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2100,7 +2100,7 @@ public void handleExternalUpload() { } } - datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, storageLocation, checksumValue, systemConfig); + datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, systemConfig); logger.info("Created " + datafiles.size() + " files."); } catch (IOException ex) { logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName}); From c1db7a5615e5f508086b00b4add75aae7d3b9430 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 12:13:23 -0500 Subject: [PATCH 062/157] add md5 --- src/main/webapp/editFilesFragment.xhtml | 2 + src/main/webapp/resources/js/fileupload.js | 78 +++++++++++++++++++--- 2 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 900ef90e9ed..8e255df9368 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -12,6 +12,8 @@ xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> + +
    diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index c6e1b18148b..fc769d9b4a4 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -94,15 +94,19 @@ function uploadFileDirectly(url, storageId) { function reportUpload(storageId, file){ - //storageId is not the location - has a : separator and no path elements from dataset - - //(String uploadComponentId, String fullStorageIdentifier, String fileName, String contentType, String checksumType, String checksumValue) - handleExternalUpload([{name:'uploadComponentId', value:'datasetForm:fileUpload'}, {name:'fullStorageIdentifier', value:storageId}, {name:'fileName', value:file.name}, {name:'contentType', value:file.type}, {name:'checksumType', value:'MD5'}, {name:'checksumValue', value:0}]); - console.log('Done ' + storageId + " " + file.name ); - $('.ui-fileupload-files').html(""); - fileList=[]; - uploadFinished(document.getElementById('datasetForm:fileUpload_input')); + getMD5( + file, + prog => console.log("Progress: " + prog) + ).then( + md5 => { + //storageId is not the location - has a : separator and no path elements from dataset + //(String uploadComponentId, String fullStorageIdentifier, String fileName, String contentType, String checksumType, String checksumValue) + handleExternalUpload([{name:'uploadComponentId', value:'datasetForm:fileUpload'}, {name:'fullStorageIdentifier', value:storageId}, {name:'fileName', value:file.name}, {name:'contentType', value:file.type}, {name:'checksumType', value:'MD5'}, {name:'checksumValue', value:d5}]); + console.log('Done ' + storageId + " " + file.name ); + }, + err => console.error(err) + ); } @@ -207,3 +211,61 @@ function uploadFailure(fileUpload) { } } } + +//MD5 Hashing functions + +function readChunked(file, chunkCallback, endCallback) { + var fileSize = file.size; + var chunkSize = 4 * 1024 * 1024; // 4MB + var offset = 0; + + var reader = new FileReader(); + reader.onload = function() { + if (reader.error) { + endCallback(reader.error || {}); + return; + } + offset += reader.result.length; + // callback for handling read chunk + // TODO: handle errors + chunkCallback(reader.result, offset, fileSize); + if (offset >= fileSize) { + endCallback(null); + return; + } + readNext(); + }; + + reader.onerror = function(err) { + endCallback(err || {}); + }; + + function readNext() { + var fileSlice = file.slice(offset, offset + chunkSize); + reader.readAsBinaryString(fileSlice); + } + readNext(); +} + +function getMD5(blob, cbProgress) { + return new Promise((resolve, reject) => { + var md5 = CryptoJS.algo.MD5.create(); + readChunked(blob, (chunk, offs, total) => { + md5.update(CryptoJS.enc.Latin1.parse(chunk)); + if (cbProgress) { + cbProgress(offs / total); + } + }, err => { + if (err) { + reject(err); + } else { + // TODO: Handle errors + var hash = md5.finalize(); + var hashHex = hash.toString(CryptoJS.enc.Hex); + resolve(hashHex); + } + }); + }); +} + + From 707709f7fcb76b312a57e0c41874c0f4605bc01e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 12:13:55 -0500 Subject: [PATCH 063/157] bug: switch to calling uploadFinished at oncomplete --- src/main/webapp/editFilesFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 8e255df9368..32739aef301 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -522,7 +522,7 @@ - + From 25c1739d1e6c25f8a96d8071446b391c4122f621 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 13:54:31 -0500 Subject: [PATCH 064/157] fix exist test for direct access --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 13c8a8a1e17..7fe8fe505f0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -691,6 +691,9 @@ public boolean exists() { String destinationKey = null; if (dvObject instanceof DataFile) { destinationKey = key; + } else if((dvObject==null) && (key !=null)) { + //direct access + destinationKey = key; } else { logger.warning("Trying to check if a path exists is only supported for a data file."); } From dd1cca07f63e231d3c782a78a12de06b32464e60 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 13:54:51 -0500 Subject: [PATCH 065/157] debug constraint violations --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index fae88a20c07..ca2b112c3d9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1230,7 +1230,7 @@ public String save() { Command cmd; try { cmd = new UpdateDatasetVersionCommand(dataset, dvRequestService.getDataverseRequest(), filesToBeDeleted, clone); - ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); + ((UpdateDatasetVersionCommand) cmd).setValidateLenient(false); dataset = commandEngine.submit(cmd); } catch (EJBException ex) { From a02980bf10151cda15283168be367f995a8bc288 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 13:55:25 -0500 Subject: [PATCH 066/157] typo, clear file list after upload complete --- src/main/webapp/resources/js/fileupload.js | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index fc769d9b4a4..3107c704ff5 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -101,8 +101,11 @@ function reportUpload(storageId, file){ md5 => { //storageId is not the location - has a : separator and no path elements from dataset + $('.ui-fileupload-files').html(""); + fileList=[]; + //(String uploadComponentId, String fullStorageIdentifier, String fileName, String contentType, String checksumType, String checksumValue) - handleExternalUpload([{name:'uploadComponentId', value:'datasetForm:fileUpload'}, {name:'fullStorageIdentifier', value:storageId}, {name:'fileName', value:file.name}, {name:'contentType', value:file.type}, {name:'checksumType', value:'MD5'}, {name:'checksumValue', value:d5}]); + handleExternalUpload([{name:'uploadComponentId', value:'datasetForm:fileUpload'}, {name:'fullStorageIdentifier', value:storageId}, {name:'fileName', value:file.name}, {name:'contentType', value:file.type}, {name:'checksumType', value:'MD5'}, {name:'checksumValue', value:md5}]); console.log('Done ' + storageId + " " + file.name ); }, err => console.error(err) @@ -163,6 +166,19 @@ function uploadFinished(fileupload) { } } +function directUploadFinished() { + if (fileList.length === 0) { + $('button[id$="AllUploadsFinished"]').trigger('click'); + //stop observer when we're done + if(observer !=null) { + observer.disconnect(); + observer=null; + } + } else { + console.log('Still ' + fileupload.files.length + ' files' ); + } +} + function uploadFailure(fileUpload) { // This handles HTTP errors (non-20x reponses) such as 0 (no connection at all), 413 (Request too large), // and 504 (Gateway timeout) where the upload call to the server fails (the server doesn't receive the request) From cafcfeeffe4d8537331e27b83d897cb5b47730c3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 14:23:30 -0500 Subject: [PATCH 067/157] use new method --- src/main/webapp/editFilesFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 32739aef301..ac4fd701744 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -522,7 +522,7 @@ - + From 8859e1dbfda1ba88ba07c00d45af4f3f4c9d764b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 16:33:23 -0500 Subject: [PATCH 068/157] catch empty contentType not just null --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index ca2b112c3d9..f3af6e86f66 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2088,7 +2088,7 @@ public void handleExternalUpload() { // for example, multiple files can be extracted from an uncompressed // zip file. //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream"); - if(contentType==null) { + if(StringUtils.isEmpty(contentType)) { contentType = "application/octet-stream"; } if(DataFile.ChecksumType.fromString(checksumType) != DataFile.ChecksumType.MD5 ) { From 25da677a62a9d4678495d3e392309d41a4beb9d6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 16:33:57 -0500 Subject: [PATCH 069/157] List any constraint violations --- .../command/impl/UpdateDatasetVersionCommand.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java index 0bcf11d371d..fefa8707c8b 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/UpdateDatasetVersionCommand.java @@ -15,6 +15,9 @@ import java.util.concurrent.Future; import java.util.logging.Level; import java.util.logging.Logger; + +import javax.validation.ConstraintViolationException; + import org.apache.solr.client.solrj.SolrServerException; /** @@ -119,7 +122,13 @@ public Dataset execute(CommandContext ctxt) throws CommandException { if (editVersion.getId() == null || editVersion.getId() == 0L) { ctxt.em().persist(editVersion); } else { - ctxt.em().merge(editVersion); + try { + ctxt.em().merge(editVersion); + } catch (ConstraintViolationException e) { + logger.log(Level.SEVERE,"Exception: "); + e.getConstraintViolations().forEach(err->logger.log(Level.SEVERE,err.toString())); + throw e; + } } for (DataFile dataFile : theDataset.getFiles()) { From e07b2df7f617ecadd92aa83bb3b976c3e941fc0a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 2 Jan 2020 16:35:34 -0500 Subject: [PATCH 070/157] try adding multifile support, fix repeat uploads --- src/main/webapp/resources/js/fileupload.js | 42 ++++++++++++++++++---- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 3107c704ff5..c3b177ee627 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -37,7 +37,7 @@ function setupDirectUpload(enabled, theDatasetId) { observer2.disconnect(); } observer2 = new MutationObserver(callback); - observer2.observe(fileInput.parentElement,config); + observer2.observe(document.getElementById('datasetForm:fileUpload'),config); } //else ? } @@ -48,6 +48,13 @@ function queueFileForDirectUpload(file, datasetId) { //check for dupes //check size requestDirectUploadUrl(); + var files = $('.ui-fileupload-files .ui-fileupload-row'); + //Add an id attribute to each entry so we can later match errors with the right entry + for(var i=0;i< files.length;i++) { + if(files[i].children[1].nodeValue == file.name) { + files[i].setAttribute('upid', 'pending'); + } + } console.log('URL Requested'); } @@ -55,7 +62,22 @@ function queueFileForDirectUpload(file, datasetId) { function uploadFileDirectly(url, storageId) { console.log("Retrieved " + url + ' for ' + storageId); var data = new FormData(); - data.append('file',fileList[0]); + //Pick a pending file + var files = $('.ui-fileupload-files .ui-fileupload-row'); + var index = -1; + for(var i=0;i< files.length;i++) { + if(files[i].getAttribute('upid') == 'pending') { + files[i].setAttribute('upid', storageId); + for(var j=0;j')); $.ajax({ url: url, @@ -64,7 +86,7 @@ function uploadFileDirectly(url, storageId) { cache: false, processData: false, success: function () { - reportUpload(storageId, fileList[0]) + reportUpload(storageId, fileList[index]) }, error: function(jqXHR, textStatus, errorThrown) { console.log('Failure: ' + jqXHR.status); @@ -100,9 +122,15 @@ function reportUpload(storageId, file){ ).then( md5 => { //storageId is not the location - has a : separator and no path elements from dataset - - $('.ui-fileupload-files').html(""); - fileList=[]; + var index = -1; + for(var j=0;j Date: Thu, 2 Jan 2020 17:12:28 -0500 Subject: [PATCH 071/157] just pop for multifile --- src/main/webapp/resources/js/fileupload.js | 37 +++------------------- 1 file changed, 4 insertions(+), 33 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index c3b177ee627..aca70f7e4cc 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -48,13 +48,7 @@ function queueFileForDirectUpload(file, datasetId) { //check for dupes //check size requestDirectUploadUrl(); - var files = $('.ui-fileupload-files .ui-fileupload-row'); - //Add an id attribute to each entry so we can later match errors with the right entry - for(var i=0;i< files.length;i++) { - if(files[i].children[1].nodeValue == file.name) { - files[i].setAttribute('upid', 'pending'); - } - } + console.log('URL Requested'); } @@ -63,21 +57,8 @@ function uploadFileDirectly(url, storageId) { console.log("Retrieved " + url + ' for ' + storageId); var data = new FormData(); //Pick a pending file - var files = $('.ui-fileupload-files .ui-fileupload-row'); - var index = -1; - for(var i=0;i< files.length;i++) { - if(files[i].getAttribute('upid') == 'pending') { - files[i].setAttribute('upid', storageId); - for(var j=0;j')); $.ajax({ url: url, @@ -86,7 +67,7 @@ function uploadFileDirectly(url, storageId) { cache: false, processData: false, success: function () { - reportUpload(storageId, fileList[index]) + reportUpload(storageId, file) }, error: function(jqXHR, textStatus, errorThrown) { console.log('Failure: ' + jqXHR.status); @@ -122,16 +103,6 @@ function reportUpload(storageId, file){ ).then( md5 => { //storageId is not the location - has a : separator and no path elements from dataset - var index = -1; - for(var j=0;j Date: Fri, 3 Jan 2020 14:24:29 -0500 Subject: [PATCH 072/157] control store via dataverse.storagedriver param manageable by superuser only --- .../edu/harvard/iq/dataverse/Dataverse.java | 10 +++ .../harvard/iq/dataverse/DataversePage.java | 11 ++++ .../iq/dataverse/EditDatafilesPage.java | 6 +- .../harvard/iq/dataverse/api/Datasets.java | 2 +- .../iq/dataverse/dataaccess/DataAccess.java | 61 ++++++++++++++----- .../impl/AbstractCreateDatasetCommand.java | 2 +- .../harvard/iq/dataverse/util/FileUtil.java | 4 +- src/main/java/propertyFiles/Bundle.properties | 2 + .../migration/V4.19.0.1__tbd_multistore.sql | 2 + src/main/webapp/dataverse.xhtml | 13 ++++ 10 files changed, 91 insertions(+), 22 deletions(-) create mode 100644 src/main/resources/db/migration/V4.19.0.1__tbd_multistore.sql diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 024281b0b96..02dd1fa09a9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -147,6 +147,8 @@ public String getIndexableCategoryName() { } private String affiliation; + + private String storageDriver; // Note: We can't have "Remove" here, as there are role assignments that refer // to this role. So, adding it would mean violating a forign key contstraint. @@ -756,4 +758,12 @@ public boolean isAncestorOf( DvObject other ) { } return false; } + + public String getStorageDriverId() { + return storageDriver; + } + + public void setStorageDriverId(String storageDriver) { + this.storageDriver = storageDriver; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index eca97e52402..a1f21c07888 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -4,6 +4,7 @@ import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.authorization.users.User; +import edu.harvard.iq.dataverse.dataaccess.DataAccess; import edu.harvard.iq.dataverse.dataverse.DataverseUtil; import edu.harvard.iq.dataverse.engine.command.Command; import edu.harvard.iq.dataverse.engine.command.DataverseRequest; @@ -39,6 +40,8 @@ import java.util.Arrays; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import javax.faces.component.UIComponent; @@ -1182,4 +1185,12 @@ public List completeHostDataverseMenuList(String query) { return null; } } + + public Set> getStorageDriverOptions() { + return DataAccess.getStorageDriverLabels(); + } + + public String getCurrentStorageDriverLabel() { + return DataAccess.getStorageDriverLabelFor(dataverse.getStorageDriverId()); + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index f3af6e86f66..2a0c9efc786 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -358,9 +358,9 @@ public boolean doesSessionUserHaveDataSetPermission(Permission permissionToCheck } public boolean directUploadEnabled() { - logger.info(this.dataset.getDataverseContext().getAlias() + " " + this.dataset.getDataverseContext().getAffiliation() + " " + DataAccess.getStorageDriverId(this.dataset.getDataverseContext())); - logger.info("denabled: " + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + " " + Boolean.getBoolean("dataverse.files." + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + ".upload-redirect")); - return Boolean.getBoolean("dataverse.files." + DataAccess.getStorageDriverId(this.dataset.getDataverseContext()) + ".upload-redirect"); + logger.info(this.dataset.getDataverseContext().getAlias() + " " + this.dataset.getDataverseContext().getAffiliation() + " " + this.dataset.getDataverseContext().getStorageDriverId()); + logger.info("denabled: " + this.dataset.getDataverseContext().getStorageDriverId() + " " + Boolean.getBoolean("dataverse.files." + this.dataset.getDataverseContext().getStorageDriverId() + ".upload-redirect")); + return Boolean.getBoolean("dataverse.files." + this.dataset.getDataverseContext().getStorageDriverId() + ".upload-redirect"); } public void reset() { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 322c9922a3f..bc15b04ab31 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1301,7 +1301,7 @@ public Response receiveChecksumValidationResults(@PathParam("identifier") String if ("validation passed".equals(statusMessageFromDcm)) { logger.log(Level.INFO, "Checksum Validation passed for DCM."); - String storageDriver = DataAccess.getStorageDriverId(dataset.getDataverseContext()); + String storageDriver = dataset.getDataverseContext().getStorageDriverId(); String uploadFolder = jsonFromDcm.getString("uploadFolder"); int totalSize = jsonFromDcm.getInt("totalSize"); String storageDriverType = System.getProperty("dataverse.file." + storageDriver + ".type"); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 9c77cdf6293..6773cb7e357 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -22,9 +22,14 @@ import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.DvObject; +import edu.harvard.iq.dataverse.util.StringUtil; + import java.io.IOException; import java.util.HashMap; +import java.util.List; +import java.util.Map.Entry; import java.util.Properties; +import java.util.Set; /** * * @author Leonid Andreev @@ -132,7 +137,7 @@ public static String getDriverType(String driverId) { // for saving new, not yet saved datafiles. public static StorageIO createNewStorageIO(T dvObject, String storageTag) throws IOException { - return createNewStorageIO(dvObject, storageTag, getStorageDriverId(dvObject.getDataverseContext())); + return createNewStorageIO(dvObject, storageTag, dvObject.getDataverseContext().getStorageDriverId()); } public static StorageIO createNewStorageIO(T dvObject, String storageTag, String storageDriverId) throws IOException { @@ -170,25 +175,51 @@ public static StorageIO createNewStorageIO(T dvObject, S static HashMap drivers = null; - public static String getStorageDriverId(Dataverse dataverse) { + public static String getStorageDriverId(String driverLabel) { if (drivers==null) { - drivers = new HashMap(); - Properties p = System.getProperties(); - for(String property: p.stringPropertyNames()) { - if(property.startsWith("dataverse.files.") && property.endsWith(".affiliation")) { - String driverId = property.substring(16); - driverId=driverId.substring(0,driverId.indexOf('.')); - logger.info("Found Storage Driver: " + driverId + " for " + p.get(property).toString()); - drivers.put(p.get(property).toString(), driverId); - } - - } + populateDrivers(); } - if(drivers.containsKey(dataverse.getAffiliation())) { - return drivers.get(dataverse.getAffiliation()); + if(StringUtil.nonEmpty(driverLabel) && drivers.containsKey(driverLabel)) { + return drivers.get(driverLabel); } return DEFAULT_STORAGE_DRIVER_IDENTIFIER; + } + public static Set> getStorageDriverLabels() { + if (drivers==null) { + populateDrivers(); + } + return drivers.entrySet(); } + private static void populateDrivers() { + drivers = new HashMap(); + Properties p = System.getProperties(); + for(String property: p.stringPropertyNames()) { + if(property.startsWith("dataverse.files.") && property.endsWith(".label")) { + String driverId = property.substring(16); // "dataverse.files.".length + driverId=driverId.substring(0,driverId.indexOf('.')); + logger.info("Found Storage Driver: " + driverId + " for " + p.get(property).toString()); + drivers.put(p.get(property).toString(), driverId); + } + + } + } + + public static String getStorageDriverLabelFor(String storageDriverId) { + String label = "<>"; + if (drivers==null) { + populateDrivers(); + } + if(drivers.containsValue(storageDriverId)) { + for(String key: drivers.keySet()) { + if(drivers.get(key).equals(storageDriverId)) { + label = key; + break; + } + + } + } + return label; + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java index 864bb700e0e..971c005a902 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/AbstractCreateDatasetCommand.java @@ -96,7 +96,7 @@ public Dataset execute(CommandContext ctxt) throws CommandException { theDataset.setAuthority(ctxt.settings().getValueForKey(SettingsServiceBean.Key.Authority, nonNullDefaultIfKeyNotFound)); } if (theDataset.getStorageIdentifier() == null) { - String driverId = DataAccess.getStorageDriverId(theDataset.getDataverseContext()); + String driverId = theDataset.getDataverseContext().getStorageDriverId(); theDataset.setStorageIdentifier(driverId + "://" + theDataset.getGlobalId().asString()); } if (theDataset.getIdentifier()==null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 7318f082d04..b108acfdba8 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1338,7 +1338,7 @@ public static String getFilesTempDirectory() { } public static void generateS3PackageStorageIdentifier(DataFile dataFile) { - String driverId = DataAccess.getStorageDriverId(dataFile.getDataverseContext()); + String driverId = dataFile.getDataverseContext().getStorageDriverId(); String bucketName = System.getProperty("dataverse.files." + driverId + ".bucket-name"); String storageId = driverId + "://" + bucketName + ":" + dataFile.getFileMetadata().getLabel(); @@ -1672,7 +1672,7 @@ public static boolean isPackageFile(DataFile dataFile) { } public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) { - String driverId = DataAccess.getStorageDriverId(dataset.getDataverseContext()); + String driverId = dataset.getDataverseContext().getStorageDriverId(); boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect"); //Should only be requested when it is allowed, but we'll log a warning otherwise if(!directEnabled) { diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 1e864c7879a..24f1cb18dc6 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -35,6 +35,7 @@ name=Name institution=Institution position=Position affiliation=Affiliation +storage=Storage Driver createDataverse=Create Dataverse remove=Remove done=Done @@ -688,6 +689,7 @@ dataverse.host.tip=Changing the host dataverse will clear any fields you may hav dataverse.host.autocomplete.nomatches=No matches dataverse.identifier.title=Short name used for the URL of this dataverse. dataverse.affiliation.title=The organization with which this dataverse is affiliated. +dataverse.storage.title=A storage service to be used for datasets in this dataverse. dataverse.category=Category dataverse.category.title=The type that most closely reflects this dataverse. dataverse.type.selectTab.top=Select one... diff --git a/src/main/resources/db/migration/V4.19.0.1__tbd_multistore.sql b/src/main/resources/db/migration/V4.19.0.1__tbd_multistore.sql new file mode 100644 index 00000000000..273de02fef7 --- /dev/null +++ b/src/main/resources/db/migration/V4.19.0.1__tbd_multistore.sql @@ -0,0 +1,2 @@ +ALTER TABLE dataverse +ADD COLUMN IF NOT EXISTS storagedriver TEXT; diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index b9df65bfc69..e553c3563a5 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -138,6 +138,19 @@
+
+ + #{bundle.storage} + + +
+ + + + +
+
From 0911571a3912d64548c8470633f570c09f0871a5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 12:58:38 -0500 Subject: [PATCH 073/157] fix temporary(which is also final) file delete for direct upload --- .../iq/dataverse/EditDatafilesPage.java | 79 ++++++++++--------- 1 file changed, 43 insertions(+), 36 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 2a0c9efc786..13290b10724 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -967,42 +967,49 @@ public void deleteFiles() { } private void deleteTempFile(DataFile dataFile) { - // Before we remove the file from the list and forget about - // it: - // The physical uploaded file is still sitting in the temporary - // directory. If it were saved, it would be moved into its - // permanent location. But since the user chose not to save it, - // we have to delete the temp file too. - // - // Eventually, we will likely add a dedicated mechanism - // for managing temp files, similar to (or part of) the storage - // access framework, that would allow us to handle specialized - // configurations - highly sensitive/private data, that - // has to be kept encrypted even in temp files, and such. - // But for now, we just delete the file directly on the - // local filesystem: - - try { - List generatedTempFiles = ingestService.listGeneratedTempFiles( - Paths.get(FileUtil.getFilesTempDirectory()), dataFile.getStorageIdentifier()); - if (generatedTempFiles != null) { - for (Path generated : generatedTempFiles) { - logger.fine("(Deleting generated thumbnail file " + generated.toString() + ")"); - try { - Files.delete(generated); - } catch (IOException ioex) { - logger.warning("Failed to delete generated file " + generated.toString()); - } - } - } - Files.delete(Paths.get(FileUtil.getFilesTempDirectory() + "/" + dataFile.getStorageIdentifier())); - } catch (IOException ioEx) { - // safe to ignore - it's just a temp file. - logger.warning("Failed to delete temporary file " + FileUtil.getFilesTempDirectory() + "/" - + dataFile.getStorageIdentifier()); - } - } - + // Before we remove the file from the list and forget about + // it: + // The physical uploaded file is still sitting in the temporary + // directory. If it were saved, it would be moved into its + // permanent location. But since the user chose not to save it, + // we have to delete the temp file too. + // + // Eventually, we will likely add a dedicated mechanism + // for managing temp files, similar to (or part of) the storage + // access framework, that would allow us to handle specialized + // configurations - highly sensitive/private data, that + // has to be kept encrypted even in temp files, and such. + // But for now, we just delete the file directly on the + // local filesystem: + + try { + List generatedTempFiles = ingestService.listGeneratedTempFiles( + Paths.get(FileUtil.getFilesTempDirectory()), dataFile.getStorageIdentifier()); + if (generatedTempFiles != null) { + for (Path generated : generatedTempFiles) { + logger.fine("(Deleting generated thumbnail file " + generated.toString() + ")"); + try { + Files.delete(generated); + } catch (IOException ioex) { + logger.warning("Failed to delete generated file " + generated.toString()); + } + } + } + String si = dataFile.getStorageIdentifier(); + if (si.contains("://")) { + //Direct upload files will already have a store id in their storageidentifier + DataAccess.getStorageIO(dataFile).delete(); + } else { + //Temp files sent to this method have no prefix, not even "tmp://" + Files.delete(Paths.get(FileUtil.getFilesTempDirectory() + "/" + dataFile.getStorageIdentifier())); + } + } catch (IOException ioEx) { + // safe to ignore - it's just a temp file. + logger.warning("Failed to delete temporary file " + FileUtil.getFilesTempDirectory() + "/" + + dataFile.getStorageIdentifier()); + } + } + private void removeFileMetadataFromList(List fmds, FileMetadata fmToDelete) { Iterator fmit = fmds.iterator(); while (fmit.hasNext()) { From 150d52ece76ca07c605c96475f05022937711273 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 12:58:50 -0500 Subject: [PATCH 074/157] progress bar styling --- src/main/webapp/resources/css/structure.css | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 819681c5c7b..07c82639787 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -997,3 +997,12 @@ span.ui-autocomplete input.ui-autocomplete-input {width:100%;} #citation-banner {width:100%; height:45px; position: absolute; z-index: 999999; border-radius: 0; border-width: 0 0 1px 0;} #citation-banner a.close, #citation-banner a.close span.glyphicon {line-height:.2;} #citation-forward {position: absolute; top:45px; height: calc(100% - 65px); border:0; background:url(/resources/images/ajax-loading.gif) no-repeat 50% 50%;} + +/*Direct upload progress bar*/ +progress::-webkit-progress-bar { + background-color:white; +} + +progress::-webkit-progress-value { + background-color:#0e90d2; +} From 92dbb2e002379cc91304b41013966f6230c5bb5d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 12:59:13 -0500 Subject: [PATCH 075/157] remove drag/drop message once file(s) are selected in direct upload --- src/main/webapp/resources/js/fileupload.js | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index aca70f7e4cc..675244d25ff 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -42,6 +42,7 @@ function setupDirectUpload(enabled, theDatasetId) { } function queueFileForDirectUpload(file, datasetId) { + if(fileList.length === 0) {uploadWidgetDropRemoveMsg();} fileList.push(file); //calc md5 @@ -59,7 +60,8 @@ function uploadFileDirectly(url, storageId) { //Pick a pending file var file = fileList.pop(); data.append('file',file); - $('.ui-fileupload-progress').append($('')); + $('.ui-fileupload-progress').html(''); + $('.ui-fileupload-progress').append($('')).attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all'); $.ajax({ url: url, type: 'PUT', @@ -78,9 +80,10 @@ function uploadFileDirectly(url, storageId) { if(myXhr.upload) { myXhr.upload.addEventListener('progress', function(e) { if(e.lengthComputable) { + var doublelength = 2 * e.total; $('progress').attr({ value:e.loaded, - max:e.total + max:doublelength }); } }); @@ -99,7 +102,14 @@ function reportUpload(storageId, file){ getMD5( file, - prog => console.log("Progress: " + prog) + prog => {console.log("Progress: " + prog); + + var current = 1 + prog; + $('progress').attr({ + value:current, + max:2 + }); + } ).then( md5 => { //storageId is not the location - has a : separator and no path elements from dataset @@ -231,7 +241,7 @@ function uploadFailure(fileUpload) { function readChunked(file, chunkCallback, endCallback) { var fileSize = file.size; - var chunkSize = 4 * 1024 * 1024; // 4MB + var chunkSize = 64 * 1024 * 1024; // 64MB var offset = 0; var reader = new FileReader(); From 216a7def0d685c6a24aca0d5bd641ab2069d0caf Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 13:15:31 -0500 Subject: [PATCH 076/157] bugfix apply styles to progress element to make it appear --- src/main/webapp/resources/js/fileupload.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 675244d25ff..fa084bb12a0 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -61,7 +61,7 @@ function uploadFileDirectly(url, storageId) { var file = fileList.pop(); data.append('file',file); $('.ui-fileupload-progress').html(''); - $('.ui-fileupload-progress').append($('')).attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all'); + $('.ui-fileupload-progress').append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); $.ajax({ url: url, type: 'PUT', From 6f56da2453638a5b2ec6a5cf1f3d7766bb81fca4 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 13:34:23 -0500 Subject: [PATCH 077/157] bugfix for failing s3 delete --- .../edu/harvard/iq/dataverse/EditDatafilesPage.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 13290b10724..a478ccfc64a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -998,15 +998,22 @@ private void deleteTempFile(DataFile dataFile) { String si = dataFile.getStorageIdentifier(); if (si.contains("://")) { //Direct upload files will already have a store id in their storageidentifier - DataAccess.getStorageIO(dataFile).delete(); + StorageIO sio = DataAccess.getStorageIO(dataFile); + sio.open(DataAccessOption.WRITE_ACCESS); //populates the key/location needed to do a delete() + sio.delete(); } else { //Temp files sent to this method have no prefix, not even "tmp://" Files.delete(Paths.get(FileUtil.getFilesTempDirectory() + "/" + dataFile.getStorageIdentifier())); } } catch (IOException ioEx) { // safe to ignore - it's just a temp file. - logger.warning("Failed to delete temporary file " + FileUtil.getFilesTempDirectory() + "/" - + dataFile.getStorageIdentifier()); + logger.warning(ioEx.getMessage()); + if(dataFile.getStorageIdentifier().contains("://")) { + logger.warning("Failed to delete temporary file " + dataFile.getStorageIdentifier()); + } else { + logger.warning("Failed to delete temporary file " + FileUtil.getFilesTempDirectory() + "/" + + dataFile.getStorageIdentifier()); + } } } From 1895611db61f302065f6ef67982fea44686ebe76 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 13:54:00 -0500 Subject: [PATCH 078/157] bugfix - temporarily set datafile owner for location calculation --- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index a478ccfc64a..49f8d070d27 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -998,9 +998,16 @@ private void deleteTempFile(DataFile dataFile) { String si = dataFile.getStorageIdentifier(); if (si.contains("://")) { //Direct upload files will already have a store id in their storageidentifier + //but they need to be associated with a dataset for the overall storagelocation to be calculated + //so we temporarily set the owner + if(dataFile.getOwner()!=null) { + logger.warning("Datafile owner was not null as expected"); + } + dataFile.setOwner(dataset); StorageIO sio = DataAccess.getStorageIO(dataFile); sio.open(DataAccessOption.WRITE_ACCESS); //populates the key/location needed to do a delete() sio.delete(); + dataFile.setOwner(null); } else { //Temp files sent to this method have no prefix, not even "tmp://" Files.delete(Paths.get(FileUtil.getFilesTempDirectory() + "/" + dataFile.getStorageIdentifier())); From ca6b4d426bacac2cc1320a6136db2ca156744909 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 6 Jan 2020 14:15:48 -0500 Subject: [PATCH 079/157] bugfix: use direct storage for delete --- .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 49f8d070d27..4000975dbb2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1004,9 +1004,10 @@ private void deleteTempFile(DataFile dataFile) { logger.warning("Datafile owner was not null as expected"); } dataFile.setOwner(dataset); - StorageIO sio = DataAccess.getStorageIO(dataFile); - sio.open(DataAccessOption.WRITE_ACCESS); //populates the key/location needed to do a delete() - sio.delete(); + //Use one StorageIO to get the storageLocation and then create a direct storage storageIO class to perform the delete + // (since delete is forbidden except for direct storage) + String sl = DataAccess.getStorageIO(dataFile).getStorageLocation(); + DataAccess.getDirectStorageIO(sl).delete(); dataFile.setOwner(null); } else { //Temp files sent to this method have no prefix, not even "tmp://" From 5d4ce21d7f4d1bf8eaeba427f3f2a63c800c2b2e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 7 Jan 2020 16:37:47 -0500 Subject: [PATCH 080/157] cleanup --- .../iq/dataverse/DataFileServiceBean.java | 1 - .../iq/dataverse/EditDatafilesPage.java | 44 ++----------------- .../harvard/iq/dataverse/api/Datasets.java | 7 --- .../iq/dataverse/dataaccess/DataAccess.java | 7 +-- .../iq/dataverse/dataaccess/FileAccessIO.java | 2 +- .../iq/dataverse/dataaccess/S3AccessIO.java | 6 +-- .../datasetutility/AddReplaceFileHelper.java | 9 +--- .../harvard/iq/dataverse/util/FileUtil.java | 21 ++++----- src/main/webapp/resources/js/fileupload.js | 31 +++---------- 9 files changed, 26 insertions(+), 102 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 54a88c27d91..b510d9686dd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -1565,7 +1565,6 @@ public void finalizeFileDelete(Long dataFileId, String storageLocation) throws I throw new IOException("Attempted to permanently delete a physical file still associated with an existing DvObject " + "(id: " + dataFileId + ", location: " + storageLocation); } - logger.info("deleting: " + storageLocation); StorageIO directStorageAccess = DataAccess.getDirectStorageIO(storageLocation); directStorageAccess.delete(); } diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 9868a103480..ce2d8383408 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -2,7 +2,6 @@ import edu.harvard.iq.dataverse.provenance.ProvPopupFragmentBean; import edu.harvard.iq.dataverse.api.AbstractApiBean; -import edu.harvard.iq.dataverse.api.AbstractApiBean.WrappedResponse; import edu.harvard.iq.dataverse.authorization.AuthenticationServiceBean; import edu.harvard.iq.dataverse.authorization.Permission; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; @@ -88,8 +87,6 @@ import javax.validation.Validation; import javax.validation.Validator; import javax.validation.ValidatorFactory; -import javax.ws.rs.core.Response; - import org.apache.commons.lang.StringUtils; import org.primefaces.PrimeFaces; //import org.primefaces.context.RequestContext; @@ -358,8 +355,6 @@ public boolean doesSessionUserHaveDataSetPermission(Permission permissionToCheck } public boolean directUploadEnabled() { - logger.info(this.dataset.getDataverseContext().getAlias() + " " + this.dataset.getDataverseContext().getAffiliation() + " " + this.dataset.getDataverseContext().getStorageDriverId()); - logger.info("denabled: " + this.dataset.getDataverseContext().getStorageDriverId() + " " + Boolean.getBoolean("dataverse.files." + this.dataset.getDataverseContext().getStorageDriverId() + ".upload-redirect")); return Boolean.getBoolean("dataverse.files." + this.dataset.getDataverseContext().getStorageDriverId() + ".upload-redirect"); } @@ -1252,7 +1247,7 @@ public String save() { Command cmd; try { cmd = new UpdateDatasetVersionCommand(dataset, dvRequestService.getDataverseRequest(), filesToBeDeleted, clone); - ((UpdateDatasetVersionCommand) cmd).setValidateLenient(false); + ((UpdateDatasetVersionCommand) cmd).setValidateLenient(true); dataset = commandEngine.submit(cmd); } catch (EJBException ex) { @@ -1797,7 +1792,6 @@ public void uploadFinished() { for (DataFile dataFile : uploadedFiles) { fileMetadatas.add(dataFile.getFileMetadata()); newFiles.add(dataFile); - logger.info("Added " + dataFile.getFileMetadata().getLabel()); } @@ -1925,13 +1919,8 @@ private void handleReplaceFileUpload(String fullStorageLocation, String contentType){ fileReplacePageHelper.resetReplaceFileHelper(); - saveEnabled = false; - - if (fileReplacePageHelper.handleNativeFileUpload(null, fullStorageLocation, - fileName, - contentType - )){ + if (fileReplacePageHelper.handleNativeFileUpload(null, fullStorageLocation, fileName, contentType)){ saveEnabled = true; /** @@ -1950,8 +1939,6 @@ private void handleReplaceFileUpload(String fullStorageLocation, private String uploadSuccessMessage = null; private String uploadComponentId = null; - - /** * Handle native file replace * @param event @@ -2049,15 +2036,8 @@ public void handleExternalUpload() { String checksumType = paramMap.get("checksumType"); String checksumValue = paramMap.get("checksumValue"); - logger.info("UCI" + uploadComponentId); - logger.info("FSI" + fullStorageIdentifier); - logger.info("FN" + fileName); - logger.info("CT" + contentType); - logger.info("CST" + checksumType); - logger.info("CDV" + checksumValue); int lastColon = fullStorageIdentifier.lastIndexOf(':'); String storageLocation= fullStorageIdentifier.substring(0,lastColon) + "/" + dataset.getAuthorityForFileStorage() + "/" + dataset.getIdentifierForFileStorage() + "/" + fullStorageIdentifier.substring(lastColon+1); - logger.info("SL " + storageLocation); if (!uploadInProgress) { uploadInProgress = true; } @@ -2072,7 +2052,6 @@ public void handleExternalUpload() { sio.open(DataAccessOption.READ_ACCESS); //get file size long fileSize = sio.getSize(); - logger.info("FS " + fileSize); /* ---------------------------- Check file size @@ -2082,13 +2061,7 @@ public void handleExternalUpload() { if ((!this.isUnlimitedUploadFileSize()) && (fileSize > this.getMaxFileUploadSizeInBytes())) { String warningMessage = "Uploaded file \"" + fileName + "\" exceeded the limit of " + fileSize + " bytes and was not uploaded."; sio.delete(); - //msg(warningMessage); - //FacesContext.getCurrentInstance().addMessage(event.getComponent().getClientId(), new FacesMessage(FacesMessage.SEVERITY_ERROR, "upload failure", warningMessage)); - if (localWarningMessage == null) { - localWarningMessage = warningMessage; - } else { - localWarningMessage = localWarningMessage.concat("; " + warningMessage); - } + localWarningMessage = warningMessage; } else { // ----------------------------------------------------------- // Is this a FileReplaceOperation? If so, then diverge! @@ -2115,15 +2088,9 @@ public void handleExternalUpload() { } if(DataFile.ChecksumType.fromString(checksumType) != DataFile.ChecksumType.MD5 ) { String warningMessage = "Non-MD5 checksums not yet supported in external uploads"; - if (localWarningMessage == null) { - localWarningMessage = warningMessage; - } else { - localWarningMessage = localWarningMessage.concat("; " + warningMessage); - } - + localWarningMessage = warningMessage; } datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, systemConfig); - logger.info("Created " + datafiles.size() + " files."); } catch (IOException ex) { logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName}); } @@ -2135,7 +2102,6 @@ public void handleExternalUpload() { // Check if there are duplicate files or ingest warnings // ----------------------------------------------------------- uploadWarningMessage = processUploadedFileList(datafiles); - logger.info("Warning message during upload: " + uploadWarningMessage); } if(!uploadInProgress) { logger.warning("Upload in progress cancelled"); @@ -2169,7 +2135,6 @@ public void handleExternalUpload() { private boolean uploadInProgress = false; private String processUploadedFileList(List dFileList) { - logger.info("Processing list of " + dFileList.size() + " files."); if (dFileList == null) { return null; } @@ -2231,7 +2196,6 @@ private String processUploadedFileList(List dFileList) { dataFile.setPreviewImageAvailable(true); } uploadedFiles.add(dataFile); - logger.info("Added file to list: " + dataFile.getFileMetadata().getLabel()); // We are NOT adding the fileMetadata to the list that is being used // to render the page; we'll do that once we know that all the individual uploads // in this batch (as in, a bunch of drag-and-dropped files) have finished. diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index 7dd16f4fa8b..5e81c87f899 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -1462,7 +1462,6 @@ public Response getUploadUrl(@PathParam("id") String idSupplied) { logger.warning(io.getMessage()); throw new WrappedResponse(io, error( Response.Status.INTERNAL_SERVER_ERROR, "Could not create process direct upload request")); } - JsonObjectBuilder response = Json.createObjectBuilder() .add("url", url) @@ -1560,10 +1559,8 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, if (optionalFileParams.hasMimetype()) { newFileContentType = optionalFileParams.getMimeType(); } - } } else { - return error(BAD_REQUEST, "You must upload a file or provide a storageidentifier, filename, and mimetype."); } @@ -1571,10 +1568,6 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, newFilename = contentDispositionHeader.getFileName(); newFileContentType = formDataBodyPart.getMediaType().toString(); } - logger.info("StorageId: " + newStorageIdentifier); - logger.info("FileName: " + newFilename); - logger.info("Mime: " + newFileContentType); - //------------------- diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 01e76d72563..6d7fc48e846 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -27,6 +27,10 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; +/** +* +* @author Leonid Andreev +*/ @@ -160,7 +164,6 @@ public static StorageIO createNewStorageIO(T dvObject, S default: throw new IOException("createDataAccessObject: Unsupported storage method " + storageDriverId); } - storageIO.open(DataAccessOption.WRITE_ACCESS); return storageIO; } @@ -184,7 +187,6 @@ private static void populateDrivers() { logger.info("Found Storage Driver: " + driverId + " for " + p.get(property).toString()); drivers.put(p.get(property).toString(), driverId); } - } } @@ -199,7 +201,6 @@ public static String getStorageDriverLabelFor(String storageDriverId) { label = key; break; } - } } return label; diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java index 6284e7fd51e..fb065dcda3d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java @@ -630,4 +630,4 @@ private String stripDriverId(String storageIdentifier) { } return storageIdentifier; } -} \ No newline at end of file +} diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index f3b389554fc..9180eb502d2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -125,7 +125,6 @@ public S3AccessIO(T dvObject, DataAccessRequest req, @NotNull AmazonS3 s3client, private boolean s3chunkedEncoding = true; private String s3profile = "default"; private String bucketName = null; - private String key = null; @Override @@ -856,7 +855,7 @@ public String generateTemporaryS3UploadUrl() throws IOException { try { presignedUrl = s3.generatePresignedUrl(generatePresignedUrlRequest); } catch (SdkClientException sce) { - //throw new IOException("SdkClientException generating temporary S3 url for "+key+" ("+sce.getMessage()+")"); + logger.warning("SdkClientException generating temporary S3 url for "+key+" ("+sce.getMessage()+")"); presignedUrl = null; } String urlString = null; @@ -872,7 +871,6 @@ public String generateTemporaryS3UploadUrl() throws IOException { } } - //throw new IOException("Failed to generate temporary S3 url for "+key); return urlString; } @@ -925,8 +923,6 @@ private void readSettings() { s3profile = System.getProperty("dataverse.files." + this.driverId + ".profile","default"); bucketName = System.getProperty("dataverse.files." + this.driverId + ".bucket-name"); - - } } diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index ea5b5ed3d5e..4346eacb22f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -126,7 +126,6 @@ public class AddReplaceFileHelper{ // -- Optional private DataFile fileToReplace; // step 25 - // ----------------------------------- // Instance variables derived from other input // ----------------------------------- @@ -346,10 +345,6 @@ public boolean runForceReplaceFile(Long oldFileId, } - - - - public boolean runReplaceFile(Long oldFileId, String newFileName, String newFileContentType, @@ -973,7 +968,7 @@ private boolean step_020_loadNewFile(String fileName, String fileContentType, St return true; } - + /** * Optional: old file to replace @@ -1474,7 +1469,7 @@ private boolean step_070_run_update_dataset_command(){ Command update_cmd; update_cmd = new UpdateDatasetVersionCommand(dataset, dvRequest, clone); - ((UpdateDatasetVersionCommand) update_cmd).setValidateLenient(false); + ((UpdateDatasetVersionCommand) update_cmd).setValidateLenient(true); try { // Submit the update dataset command diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 0cf8c801068..9cbedb3ee45 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1207,19 +1207,14 @@ private static DataFile createSingleDataFile(DatasetVersion version, File tempFi if ((checksum !=null)&&(!checksum.isEmpty())) { datafile.setChecksumType(checksumType); datafile.setChecksumValue(checksum); - } else { - - - - - - try { - // We persist "SHA1" rather than "SHA-1". - datafile.setChecksumType(checksumType); - datafile.setChecksumValue(calculateChecksum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType())); - } catch (Exception cksumEx) { - logger.warning("Could not calculate " + checksumType + " signature for the new file " + fileName); - } + } else { + try { + // We persist "SHA1" rather than "SHA-1". + datafile.setChecksumType(checksumType); + datafile.setChecksumValue(calculateChecksum(getFilesTempDirectory() + "/" + datafile.getStorageIdentifier(), datafile.getChecksumType())); + } catch (Exception cksumEx) { + logger.warning("Could not calculate " + checksumType + " signature for the new file " + fileName); + } } return datafile; } diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index fa084bb12a0..c3da541414a 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -12,21 +12,17 @@ function setupDirectUpload(enabled, theDatasetId) { fileList=[]; for(var i=0;i console.error(err) ); @@ -123,14 +106,16 @@ function reportUpload(storageId, file){ function removeErrors() { - var errors = document.getElementsByClassName("ui-fileupload-error"); + var errors = document.getElementsByClassName("ui-fileupload-error"); for(i=errors.length-1; i >=0; i--) { errors[i].parentNode.removeChild(errors[i]); } } + var observer=null; + function uploadStarted() { - // If this is not the first upload, remove error messages since + // If this is not the first upload, remove error messages since // the upload of any files that failed will be tried again. removeErrors(); var curId=0; @@ -170,9 +155,7 @@ function uploadFinished(fileupload) { observer.disconnect(); observer=null; } - } else { - console.log('Still ' + fileupload.files.length + ' files' ); - } + } } function directUploadFinished() { @@ -183,9 +166,7 @@ function directUploadFinished() { observer.disconnect(); observer=null; } - } else { - console.log('Still ' + fileList.length + ' files' ); - } + } } function uploadFailure(fileUpload) { From d72bf84ade3a1e5505972fbe2b0ac557c2f8d53c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 7 Jan 2020 16:48:52 -0500 Subject: [PATCH 081/157] remove tdl customizations --- src/main/java/propertyFiles/Bundle.properties | 16 +++---- .../images/fav/android-chrome-192x192.png | Bin 3677 -> 4055 bytes .../images/fav/android-chrome-512x512.png | Bin 11159 -> 15242 bytes .../resources/images/fav/apple-touch-icon.png | Bin 3427 -> 3170 bytes .../resources/images/fav/browserconfig.xml | 3 -- .../resources/images/fav/favicon-16x16.png | Bin 553 -> 814 bytes .../resources/images/fav/favicon-32x32.png | Bin 896 -> 1190 bytes .../webapp/resources/images/fav/favicon.ico | Bin 7406 -> 15086 bytes .../resources/images/fav/mstile-150x150.png | Bin 2503 -> 3372 bytes .../resources/images/fav/mstile-310x150.png | Bin 2767 -> 3638 bytes .../resources/images/fav/mstile-310x310.png | Bin 6100 -> 7163 bytes .../resources/images/fav/mstile-70x70.png | Bin 2056 -> 2347 bytes .../images/fav/safari-pinned-tab.svg | 39 +++++++++++++++++- .../resources/images/favicondataverse.png | Bin 7165 -> 0 bytes src/main/webapp/resources/images/tdl.png | Bin 14054 -> 0 bytes 15 files changed, 46 insertions(+), 12 deletions(-) delete mode 100644 src/main/webapp/resources/images/favicondataverse.png delete mode 100644 src/main/webapp/resources/images/tdl.png diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index 178d47b6041..b9a123fd517 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -122,7 +122,7 @@ contact.header=Contact {0} contact.dataverse.header=Email Dataverse Contact contact.dataset.header=Email Dataset Contact contact.to=To -contact.support=TDL Dataverse Support +contact.support=Support contact.from=From contact.from.required=User email is required. contact.from.invalid=Email is invalid. @@ -289,9 +289,9 @@ login.System=Login System login.forgot.text=Forgot your password? login.builtin=Dataverse Account login.institution=Institutional Account -login.institution.blurb=Log in or sign up with your institutional account — learn more. If you are not affiliated with a TDR member institution (see dropdown menu), please use the Google Login option. +login.institution.blurb=Log in or sign up with your institutional account — more information about account creation. login.institution.support.blurbwithLink=Leaving your institution? Please contact {0} for assistance. -login.builtin.credential.usernameOrEmail=Admin ID +login.builtin.credential.usernameOrEmail=Username/Email login.builtin.credential.password=Password login.builtin.invalidUsernameEmailOrPassword=The username, email address, or password you entered is invalid. Need assistance accessing your account? login.signup.blurb=Sign up for a Dataverse account. @@ -303,16 +303,16 @@ login.error=Error validating the username, email address, or password. Please tr user.error.cannotChangePassword=Sorry, your password cannot be changed. Please contact your system administrator. user.error.wrongPassword=Sorry, wrong password. login.button=Log In with {0} -login.button.orcid=Google Login - No TDR affiliation +login.button.orcid=Create or Connect your ORCID # authentication providers auth.providers.title=Other options -auth.providers.tip=You can convert a Dataverse account to use one of the options above. Learn more. -auth.providers.title.builtin=Admin ID +auth.providers.tip=You can convert a Dataverse account to use one of the options above. More information about account creation. +auth.providers.title.builtin=Username/Email auth.providers.title.shib=Your Institution auth.providers.title.orcid=ORCID auth.providers.title.google=Google auth.providers.title.github=GitHub -auth.providers.blurb=Log in or sign up with your Google account — learn more. If you are not affiliated with a TDR member institution, please use the Google Login option. Having trouble? Please contact {3} for assistance. +auth.providers.blurb=Log in or sign up with your {0} account — more information about account creation. Having trouble? Please contact {3} for assistance. auth.providers.persistentUserIdName.orcid=ORCID iD auth.providers.persistentUserIdName.github=ID auth.providers.persistentUserIdTooltip.orcid=ORCID provides a persistent digital identifier that distinguishes you from other researchers. @@ -355,7 +355,7 @@ shib.welcomeExistingUserMessageDefaultInstitution=your institution shib.dataverseUsername=Dataverse Username shib.currentDataversePassword=Current Dataverse Password shib.accountInformation=Account Information -shib.offerToCreateNewAccount=Contact your TDR liaison to get help and training. Published content cannot be easily deleted. +shib.offerToCreateNewAccount=This information is provided by your institution and will be used to create your Dataverse account. shib.passwordRejected=Validation Error - Your account can only be converted if you provide the correct password for your existing account. # oauth2/firstLogin.xhtml diff --git a/src/main/webapp/resources/images/fav/android-chrome-192x192.png b/src/main/webapp/resources/images/fav/android-chrome-192x192.png index c4dea7054d4ac6f9ca505f34ca0a71b33a9f6f05..f824dc8fdb2b318e5ba9fb600e172aca774861e0 100644 GIT binary patch literal 4055 zcmeHJ=Tp;N)BZsq^mx;Imxv<0_bxT`B2Air!9+?hAT@vpQUs+)2?$8=P!7knX_l-+PSvO&N(~#(!@xIhLVjE000_2T`e<$ZvR)vL4@0l z7%NRsBp&K8bpWVMq`Gh>C14&`T{9Q}1VRA-^Be%q2qw%r06Y}~fK4X=P)G*=W^Zh( zsS*GXUz!+NXkTAnf3*|*-}zq${@Xf0c=^TPHyeZ>a_MJgWDeZWaS2YV?p+*h&Ux-> za+Bk6>lr>unusZ6U{#7L^q?F{@@VZmomG44PdLc$vgIGLJOP%T-&(g;ejPc{&P0eX z(x@UnYWVWmC$D>8sLht{Y<1W3fO8L-+arI_7g+6ots*v%ReF#x{B!QDglJdTC$Et~ zBrg}|P_$#I^D~uK_pI+$*q}`*8eU{F1h}8J85Zr8knZZf-!_*!woG!<`#fQBC25pD zT}dgn3>WA3!CiFB#<}jO#n&o+@rQN3uIo!In?E7H{dD|IR{w{Z4T#-X%6GxWO-%>& zph>nsd40NKRN*^ib{S#rzfIp3gp9E-V{H{uW4!NYKcPX!*by2*Jk8j zx2_F}FeXa=6{4M8bU49C=BEeKCR+y4&??d?^2#&;0HwH|mbyj23_d$3n6aC&hgrjr zUWk^iMNgbBd>G(`Vi-W`JQ~`(Nqdy(2j7rtf{a2Xj3&gf6gNoYDXJUVp1{eMMv`I} z$VCm*pRv$1L@;Ov^Wx1KCz=lb9)TbF532kg9GD-l3A=V5%-C(2&X}H)M_HLh6|f%ebfspl_l#$K@%qMr|G5Apc&r3Xx`wX_PwH>|Jh7(Xit!=62mz zZWZ(x*Eqw*54j}AOBx;&xwgp4RH<)-93?HC$n4Ax;R616W%(>%NJ2{}f&Tfy{kPIN*jY~DU$eT{mj$HV}wA`=`5-M`9DmX0K=Szu_yvaIrM6cZLTd1A>zweiL@`9r1(wkZ9r+0J+<(1x-zYl=xz@L6Uf0b zgc#EbN*rg-a-^`ddEnAC@Lt|;XPD$AVG2q1I$1j(CG4YkP= zqbG_8Z_BI_Kvyf)O>yWSZhTV6>XYEhoL9L&HlsWS+*_wDCbz}@rcySUge(sn_ug?) z=QFJqo*Gu06N1at9XW@*u|J!T8t&{lBp244RiG_dw|P8n6MFtibsa@B?D&Yap*D6s z)r2)ocBe2K@Ad-ctsK@j8tc+o!RCLrL*dU|kFLZ(Qg*z?ZQ+bl|4*?ro}~qImd6*u zA6#9*W)s}VJ(R37{|tO+9ymoFw=B^P7&V>?*Nfh8suwSF{MI2c20)yXYs>TZ_8yO2 z`)R3$U$JJzS#Rj9g_A>_LHqsY`efeJ>^f>Z3!|K@8o_}nkX&V%1k552%f^p9gmm7! z$d+S?x%|luV7|SGcGyP&Lq8=&OqmDL73}=FG-)apyXLaI*FP2sapIPFg)tf>!gdmP zoPL0p%NV&JEj4i(?4P%#5T(Vos09Ap4;#Q{Eu5qw9kM7zNRtCLu;Gw_;07_K+w&*3 zX$RZ+An%1Zc+h1OWRZtJ$Q&WMV9dIwwu@2U3lZ{Jv<|CD$?pD2J4~$rOk%m0O~@30 z93}q`k7PL27&Y}{X2+T0f1;-3h;6mW_CyE)m_~_mZy8dpJ1Af*35BzfgTuEBDp*n} zjzvcjugD)!a$=|s99pCkAnl}Nr_ItC5T0lD-(USj@e9@D1&k-8mz-X7R)&3nx|-?j zJnqu?TtjtCHvKx)@g0n~tkwOE5WRO*??)$)vx%RuOi7E9=7M{N*YMQ|VelMX8CqR{ zk!Za<3QK0kUmdrqgd&>SlyBi2KmAC4W#uoa8X5^{Ezv`IUXz&_55NVUU3O z4OTCSwKUi7MJQ@P<<==KdSUo}Q3paz$-rOTx`OCom@3o9lw`1G3eEH{73Hr#>0}JN z5_pN9A8u-=D!d>$&R6cUx!kry*c}P4&lKY>TJ+zC&e(c{-j})7p8J`J${4zmoHO}+ z@O%ZJbh_^S5lsqp29)y-w0EzW)OLAaklx4k~VJ>sCC;v0~3j5gB;7MrI?i|5-* z6K|UllNW!qUmKs5naRRR>4+WvdAeDt$YL5_P09ghNS95Md%8v{`$B^9Fch37zSce2 zzNN>?IJod$@uIT@H`bua>Sky)#tHHNeWReHG3lRntJW_(9Id2^&m}e^Bb5_)#&d`Z zW6-+fK!N{T#S_`r$FhzZz;&+TfY0w;s+olzI^z2p{>2DJWDsC7B$;~d4p3m2Lp4k&$qjm6@ z4}kS6?iyB7RNNPAGaHe;UjLTzjn+BA-|Lo(C5d-KR}H4B_XPLa5wEKYNP#t#{kMIN znyS}tm@v$36SrGu|NhxM86;Qne6-pt@vh`Hw|HaM=Qx}378)OsG_KxaA^rjqV7Z#DRj`}>Q)pFBpnI(xf`c=>u{{Z?WlkgNkO?)#ZJ`}27Fdbz_9ZajX^yxn-< z-hQqC5P)4ipeFaR5)|w+?jO)wH3LBWR3ffa^!)S&$qXQR9xaU2mxmz!dJCTZsqt@J j16>?lWc?W3TzwH>lbP1pFy--Y4 zMgRZ*A%~?0dz>VTrvP`EC5@>7b(jEYkpp>~B$Tcmg`*98od9%~6MmimZj=CWmL`#` zC6lfrmaip`sv3f!7l5DuYLgn8vH)z86P&aln6Ut#whw-u{(t`dQlPjAowXd9vHAS{ z0B4Zy_4t9f%Rra0^!fTJlC5^L#>(RAhsD#t+~wfz^5^mNm&VjVowcjc-Cd@=w$|a> z>+sU%?P;vRq087ck*$or&oP;?ddb%-jjAk(r$V;KX1vV1Ibibu01U=SL_t(|+U%Td zccMNM#*6rfRexKBuG?GKl_#MpiYVY;+wcFeZUP7iB!aEb=A50=mu;P&b0ssG3_Lvi zvw!x_Dm2BU9AIh8=>~W^I&>>Ydfw8lpkt424ZS`0HY=JHWIy+)){uF8?oq8F>nQ-B zSVOuE0LD9N0rtn|0HCLe6e|b-Y}OPj2mthU6e}nMpnp|E?o$jP(WxQ*Z~-_jsRVF< zfuCI(1XFD(a>l&JtP9iB7f)^6f{XK5CDob6e)$oIpWTYX~SPuOTD^KUzbPg06?)M>pCXjUk7C1`Tc52mmIC7@n!PtO*nZ1Dv$f zHh+-qfAAiJ0{9Nrlpn1im_VcKf?hN@1_?n+qw8vKtpWcL0*R`Nr5zI*vJXiPZ3@bS zpk=QpQV@p;LF+)xyxERrV1R_cP-syQ&oGz}OeU&Amx8h(m`waiJBngyyZ(QcXO1I; zU}05BAeA-Sh)l(rFL5bAj=+ToR+j%()_?vhH22{wqt*fT64-P)9ReExdkn1IZZ?}; zV2wr}*lac!3+cbW1vyBKsZNpVAheVqLHJ#W@D4Kl+~s9FK!2Vf z253M40t`T0$4FtP7`@vFNC@w7p5IF^aybAv1Q!6IOb8~e4OG~z4`m6iBD}BCpmyaV z4Sg{L6Qr=i6TGGbWf0z>Dn;C#8pKlrYZXv*lXtMAj zf@K*36d{86AlJ8!^g>?7DM16?h>F%RN&pvjB!cS*zdx?k=ZfFxd?ExwiGL8p1a4YX zh}&a^03QUD;p6)GLR@+s5&*d(!e=uqfwwI~Kt}i|JRhwO!_FsH0e@XY(PrvF_hh1b z?3G3MRnmV@-U*%0g&?;PWeHlXg`~pmuDL}f$RqqJ<$stGzP1WTB06oen|Lr2;`LpC>y%*QJgNNK-;t9|wj9aMa0|LQsMF~1S#vpAZ1Z;5JH6B@eaLV-4ceSLNM_0f4?-eTyaez zKoRcZe^}m?645|gV^h|NODj_h!DMn=5D<>{>dmi>?y!uA+<$z^(ir6WFGnz$Xuf+4 zgOEcwx_7%<2u}P%TnNyeXn@QE1u5%r%Aaqb<3V`BQTGq#@uDLH;kdo$oTei^WUSC(F`6@Uu$Fn zwk5gAf-x8Hjv69V;(xHKxcmqXL5^VN!SfMT_H>Y>19oFUTW$yd6IA7Y*wYNw^0j3E zKlZuO_#6R1U^E};&xQ}r!3fp)5AL`c*9c}vGV^oCe1CAt^i&7+Z>Gj ze0v&YmO-+R0|H@-3jsvn8T+hzJ1DIM{#QQi<=9Q~LhegCE(9?GcZzq;96*9l$FQ32 z&mM^);^5EUdoTgVCOanvAZ|X^GHfR<86rU+PG+JCost=8i$5&v(&D17VL|0I!Y_kh z9h~7=xqtmBCg8b^i@NWad4Sr69hco~Hh>6Jc*z?=M4z-8mgKIOK3f8iA-f zs?GmEL8P)SB+y{7FTH~~&#EWfQWF3-E-&gEm48)-832?(fR6=wrau>`Cpx5Y$#!Mc z?Z4s;lfw;;ph13}jp!JsctDU#e#>I&^&co`bd`W3Iipxwa>_vfm^%Fj3c60vz(2V< zUrKQIT1o-x^+pmjIeymsn6B9!q5wF8?nXh> z1K%Iy0BSZLx;sllkR=!-$xPo#>4+>$@J#@bf(i(_jjlb9)pOwbg8>fV{5? zN=MQPR`P~TM%@vDWJBG42Y>}nbP)+IOSu6L>*s>-9RPEw^8$x}ln)>RTo59F?|A#% ze$aXOBtZ^AfB{6YJ_UV}APqvi6kq_wQfg9=T%G``?XW=@s8GxJysS$>om&ZrL4Oz+ zssB9h*82lP*A5dfo3YcfC)l^FgrvGS2w_tuB9?^ zg!DlP96_289vj9QjEZ=Iy8tu=!XD-|Iwye zgN9-S6%vfv=^vz zVyMmza_0)cOSkw12I5j{UO~v*(B9kDXDmQW&^hnQ9H~G`$RR)sRcdHA>~u&3onhGq z`P^mNR}iYy5MDtcL8tGR|9@bR5-P*SE2wzqE*CYm0e?iBz4!Y8bxnpg|7u#W`%U|T(7yXVWVYp`cKH8jwz40zF9GTS z9#dJ`q#$77Ps8IY6%AOS%Yk{QoQ7a2NJ&(WwilTg%dtV0qp}##AO%sk@W)jp$Ux>7 zsGEi?Xi@m%tP(&z;J4PLb_!DBGN8{rIFPMs9MoLLV#1_En^nReM|gp@s_~mrH3cbg zSkUv>Q%zNVJ}RamUgzLZJqgLS-owLWSvMvSgW|m{K8<%2visk+lUQ z6-9^)m1Q~=F^NLf%yZq&`Tl;-^T+c#uk+H}_h-32*XLT_*L7dozi+n)L7sphh=?`W z(jGyu@J}ov$Pa&}L*6aGA3mQQdv_p6d4kZ=F+BWM@+8~uMUb!!2tvDnAm|XM23?7_<#w#;6HA=+Y;fS|7F+ZC&QCf6teSK1QC!%|Hpj3eR3Hd3Itj2 zwG#M@6Oq`qdYiIQHG*vYX>GZ~G33MeK=4y9tBmEJ9jwG0zV(;Om^)MdQn|SCRT*K! zk?8%jRckQEoS)PDL^fW2tnu2eSMbHIHqH)2y4W}Yh)iPDeZXHrYkHN6_p-i zc=No^kGg?~`ksi)SuWL`78R1&T@#pV$RIruFHsc3m{Lyq%3zvQWp@Nc7-& zud=R|8eHwpVR@Sz6udvH$4zjw4BLk{kKIN1@yHwz#aEZNcdLpaW=Q)lJ>DSc3PV_I z4I<~OUfV;D!ix&H*+dOb*q)K3NQ+0b1v|w_BsJz?m2hhKEo(y7Q)DMe9#-5vBCf(m z_8sJpEZ8P&>80<8czL|5`KGw)sZ3uB$xeA4u3j{JusJ+|Dk~wUpn=#w4NK$sazZ%o z*`b6PzR9B;N8;2t``?yaSk59VKcekD(3N6~H@}=5?J8s} z7ot{nnVN~IlAY6|JWQD1#nn}@U$+KP-$e)@OdjGGut@CC*yko3A(mQr+u4#P26?L^ z!W2}mRir<_JaO@nvk%5L&!W%dqBCJ&AjXir(h zb$7;`=YLhidaqt9<}4tQwA>{{o3U5x-0f zixPPpj58xQmU;ja!cf`_)N8$I-G%37l0ThMp9~{T<)GUH!!~MTftP%6ftBql5oZ)9 zHHcGn=$kF@=9GHvSGo}GrmIg<{bR+x0rZx|B}Ov|w4mH|zs&@UStQ$pHv$2;6UZ#K zV%NMgUkQJS8*l1WmJiMqa|hRC6cve^v?oqAp(}3FK)P;7^&c*jju&|-Kp$lh3$*79 zXs!`$LXX4!Q!$yl5As*v`w$jv=i$kf{LvLizqy!3tbv&5z`wf2-J*f@*;?v(S$E!qSo(C?wiCGtc8(sCh}Bbt4CR_Bi??*;2C>lW)#?$FCa zI(hv!i-f-mX$EHaDVUCUn!X}kVC+eUwaIlgyr9?h##(%lNcumX%N5kLSD>*-y5Lhu zr}v}|dxg)Q%SBW{iCeOuZZ!*qkx|TktJ713_2W@5o5SO&X>M>gxTUoe!?IXvrd}J) z?Yn3FYq-TLSp?b@Taa{$X-{X6pEPC&Yx>EvzLm7cjLBhVYhcwdo191FUih>2<=Ry+ zS{!V$nr{4^PQE3>E2bTdXq%8_$D7d-<_{33nrR~rYs$M$?>hL*(#zbJ&pcHFKSFoM zIh0leg~-k+ORP%`yy81rO4?LjqMjxAUW+mz^S#J&#F?(s7mW8zelvw*ra;m1M8 zcw%pPyZsZ9zPHUL-RSA|oWlG3HTp&->V-)DiXbHVi=x1bhsYpMjp*V?}0oPjW-@bYVS=S!o-YE1o-j2tk& z5Aw{ya}wqW+~m}^^J#(jeW&Ki9@sT*uH(0|JgI?br9^)e?YrM-q0MJ{EMoAXqdRh9 zbI8**bXNvbJhpT$Lf_gE%}Nouyomr&(660R3J)XQk^TA zdS(%;o8S0(JTv-Z=K|B}0~w#~ZBKEx`@5`AyjsAW@~(962Uy4 z$;s>d9yflAJ))V+{xT7e3{+TP)o(;+ekR2rdEEyozNQ?!jjt zy`9c$=xbi0&8%wuGM6@j9We?am*T|z(g@~H4Cj@LKk*uSzf9~`MN~w9ZI{B9dFw?K z1J+UQ&TO^qjTGu1>MUp5HTnlcJX7(9YgxAzSG+l>8Lh!PO9FI@uf@k+px~ACG}rrk8Iiw2XX9%gB-y4#=W7dizuFooq1)`v3Y)5k0Fh0 z0k?WAhCd-!g7)dGz@()`v&+zem(_=z_-w^Pf!~ayX7zXj3tqW3;cY^6S9#JAhdT!^ zp;T=%oVgp8Jafrr$Z+P#L5#Q`1AVp`J6VQ4GkMo}#u3N_=7<8B@7Zb^5eCI~KNlbj zLrYlt_QL*%wgt=Fz}p%~u}f9~*%cTPvkPIm%45Xk^x^s-wi&qn{{OX#NbyO6N66-b zKMe6oUE*tK=hAvR&!pCv9vHo|MxM0KVXf$YbZrhV4h(E6F1#DMSM}1$Zs6@*fr>oo zwJQwWW_S=eR1Oo@u0fKNbAZIhajuVh-evry@h_hQ2A-`S=n{mhL7A3EU~NF@pL!a5 zEeHGW-o8bMi4H#B6%*HeNJMdWFy_Bwh1m=}WDT>`Hr3sLaL+0KZqw$OcRP~UAVqO^ z)|#-{@Cv<=%2%^RfO8|a7cvI_@pSX}KQGqM-q=$fD^kcz&LR9czi$-zWf{Lj zRDL$&k+K@}(x39BedsI?s02YVf}wiS33G+Wp1% zTHe2pSDOwu)43p1{OYw))uuk9`v^a_vVMyw{#eCv`A^Qz-C}dfqhX(3(Yud27@H#6 z&Vtr5_kxJ!zQR&`3Hv4d8-klH-3Ukl+Eqvs? zHyzIVu}30F6eM=qf%BF>p+=8#VaB@kdM?Ke^R(za)`0R^gezdb|O~3Ev z?rycAkKN#9`OTH}E~MT0km+K%3)J>OUOvW%&-7$OmwsMw_xG1(olEEUTathkyEcSR_=R()9wW&2r0DKstWZ+)M}d&!#=BK>cF)1&#e*-vXzLqK-vlF4Wy-4tZ-@{7 z$mtBm0z(_f9}5w3_t^&I6b6EBEg$fE2Pfu&H+UR)Zp!t{3pdG;w^RJ3H?!zv+jX*&G(jYvy*tQRwQn*}9&4Yud?;`mjSHf;UzKO_hj zz#1ffJyiD*H_{`cF?v_3U*GY`k8wZQ&$gz+ai#7;5}}s~w51q`kBZdrF7G87GckV5 z;C%vvz9CE7Xm+j*f-1j)9He#JPG{^OT#x?9H)rPI-6T7L6PSmHK5{$1Ga*2uM=LmP z?e5Uh1n>gA7M7X&RFPkX;MP5AlqRhoV6?3px3@0L$U3Um9xS+=GV_9$|9CDw#TswE z>#RH_=S=JXt<`~YXcJrD5Fy|w$DZSUC8rHD_-`cVh2 zt=UY$msL=uIg&Gs>kKI}EOjtY)bZKWhb`Fqdm8JyN#xP7d`&m}>3|G;i(`VEbka^g ziR>%t_*n%GzlS1Mv+67@crMLnCB6+azlw%hZP=ltB|^x>gwP~dbtNJW374I_N@a+)Ivkz*oA5(~ee8B>^!fEY@7kKx zx&QFCaP}Wvvf>B-;B^FIMWh$Yg-rNG2hB60t;R)QG+(O!B}=A!peImMh<+r3v|dFLJdWh` z=*7P%52@GII|-%>p~+7&H7$V2sm2b9@p%+cGuO|ppVX0Gkdf2WK=hSoV#3<-Hd*K0 zg^wUhW*iyQVLY4uGAHuyp+VL*`XBYEjD7rWu*efQCz>qXDhtCebkMh#R9xwfV9nAE zBHE%VMGbB*oU2!*Sri(X$@2wcbRZ1(hQxLQKph*_gOF3?%3`Rgs}>A;Vs|!RmpwM? z?~m#plIRmKrs-Gz$Y8|L22hxR!PtXlhAxDF?VK9c3AXbFKk1(FtKDN+Tu4F}&txw| zH;D91E)P29sGV|YK#~mw`kI@=b5YtL*cyeDGOGaMhLVbDmI+7N<0T{OeBtY7n#8H6 zdTw!FX+2{riLYxfubDRRxHTF9ja;XqJ_Tmrk(6q z>=~PKYa}`0q;YZcX|^Hc6h;t;c$f;lW>ZhmVpVAc*Q5QDeUOo#H67kfJB*lBx2q(d6YAMu*6WI*41j>tPwI8#e&7rjvz@)3 zMT&kO^Q<30afSpoLLSQH4DyBJoO(#{nr~G&W;DC8MpKn0JG{9(SkBdR%4Ky_X}O*l zma^%vy9+FM!k1t!4~gumzBO}?snVgbeBTS=8^})}EtS6+y)EsejmKuLrdEyk92P*I z(0V|V{89DHSyl-?|=PM#x9Z3ZOad2Al?{6fO~36CutGG~Gv>f$9zSv|QE z6yU>NRpdCBJ?Tk5&Uit}?`|D>Fqm!>?)eS;msPJlTJT$>?>(tC@eyt=G4mfam5gbB z+smIkm$y%|N%ta@J<^lQYcL+3LfF~R|`#Ct)Awa{bWSxp5Ws?~# ztNlc1!hFOjEOBa<=3lT*e_E*M>gg})4xRM`75#l+9q14n5smqb75dr%V>A8Xxm*CL zu{wYQ!WX6b&pBFg#62RoCb5yDo{<;-_Xfa~hb%YNCVE6crfFYZ;r&~zifLna31@i# zAx_R{$DGHW?=@>DIpC_N^mr(kk+K(W?vm@arA(#_SGK-K5zph(^vNiV8yxjqjzyP+ zGgP{5_b*eoTNACt<+){yCB3j)E+GIU60A_q^^A=FZ&V8(#26!mJIjQ7(ib&3+8#%_ z;Z<^`!=D2JHu*_)2mBPKnIaUkm&H~@EMd+4Zh3K#x zCKf5XLq%5hCI>Fj$BvogPO>qU%+nBhZs&ML3fB&T)jT@(WUZy;u3U&Vx2@lcDCdufQbmp2w7v|SiWFFy1K?3#2BCARq>E1WB?7^FsHm|TUE5z(}) z*Jb2e*;zsW3O1=jxbmw26Om-emf%OT*Q(Yv+}Z9HJdP$JmepWI&D;4I4|^Y(m0x6B%$b({~c)wK)M?g^Z*U z9D}hPMxIEi)S7&$UMm8U!_ZoJU?h@#07*)9fD{U{L57qSkSf=U*g3_0L+|Ntpg{zT zRgMNGw+~RNXgGU0nuerb&%+!-&OL(ufxw5M!xjuTHBMklEIDY6_n7Xjj9ZiSKOVE1f4Bk0L)T30rX!1nZO0q z{3jEJAfC@r9qHKD=zjv3ar-W;UJM#454;coid=KoAlqfknd@Qh>7xQ|<-iJn1hmj` zEzM7nxftNZN3fBohW!B;_c*F7-fv^gRZ8LN;Mc#b#KT(W54#A+qKw3&gv_T+2n1C9 zN60i7KC&DLrxOHmsJ$E2furlAGL?g)l5q)|@%;dz1aJD1@Z`ElDp~~L= zf|dS+fP*L_&0yfy+Giy5IB)toRpkODk_sL;^>q(Bfiy1FX6G&v*slwN(G$Z zNggN92{?(Wqq;5J=P8V<%m4koa-aH@K=^BjuvM<2qgq8+pwg(mqWFclI=N1{bE5QY zP~Qf8&13WosI>n=)m3;>1*PmfN|`z+92>(BhO18c1tgG%K$oOmHZO_mHi2yT3;v~l z0o}&23nG151+cXaY@U4{YPjUhy}s?K5>-;k(I`cpf=RLUkk)N&e##;xGJp>61F*K)cG#N*b9CA^bQ|q))-?0w`}oCr1G|%G|GB zdl)0*JfizcNjIZNcn?Uomm=A#59O)`zDABxOue?VfA3NL0zzX3n{LUU0av^OBu793ULQE+Ms!-FOvARCY=Ry#s@b;+HY3iV_2nv?nyd^sk=nP+Q`^2xcKk zs^5rpccxe~Il&7NQ zbJUh3qQRfw`6||HHkNl(6UZRj3~E*vX=~T5tgyP{f@gH5m!c)TB!?7|Fudk3u%7 zxQm&EzAhLnogi~!SPSe>rj9;ANbW;E_1ZObJ1mc1(=%iDo}*$q6Llo~mQG-vIW+p%Kf_LU38T3htOZr66HG5SX*U25-&>{z537FI?DZkdz>tvPkE6Epc2?-y8qR`_u38j#vyOV11sqV{q%ai^7`)V!*#DCz{R6;q5WrY`vky!f$4z-_`a(@)Xz1K4o1 z9IR9{@aIrjyM0+R{SIAD(QFGYD!;yOF>U0^EUzSz(Ut7&LG|$TD881ZMZa2N_n6X) zdGJeb=oG8~?lyKO;fFbA(f8P?d{??{p|-A_+xLlSy)6s7v!Y!u773olpBDMf|5SFo zxzmro>`B7>4z5zyI!FK~p@z9%YwpydXZS?+V~(9b-eTIsLs7%jh2L#QX?0`7ggnVe zvG3!ak!>VKB}5d$l^|pV+9AcxK6;ZIDWCPLz)m$_|zP9l`0$%RkAC za74V6Kw$;$XK$B@;}&Qy_y9>2qTMM{{ZgFuj@}%k=wht@q&IQu3|rZ&7TC?QZY*7O zg#WZpMsCMC{(6!0QO|lPrdOdLGM83^)v0OkrX@$T zEE$>slqpJ@Qa0jiBq{on{2ox>HYIg6FYV=u>!$-f9`9z;d5kevj_!ic*MC^<)i44yj^Km}StROy$Hu1>dw_%(@Z*;^|24HNM1Oz~-)cov+fy>f{#4dcQ?>oJ z)21&FXi|RU;Xe}?(Vu$mCloC~Z@dW(m)_`5LVmLb}Z@JFzZE7x|HN5lor=$S#k z!3HR&o-E4Oy6>Oiy;}R1-UQ?%{0Vso^RIk^G#Q>lMB5_wAG6x z!jG#UW|NQzUrvl(Jj^-5RgkPq8yj5aRf?+AB}JbX?ZLJymR9WPbjN_h@>jIvkK2Vj zKLX(llp42g@HG_=`bpFVUq{sN$m!RzN@F`j4bmz9XVf-+}8t z=d5uRVN^ZPGnV0Pqswsr84@)ewlW>gp(i-|+@cN8RZA>dJURtwY1WO31WT^M=V>Ld z1);H22!RPdDp==B&tB=qvDg42f+2ekrNRwRXGZd!M`ZKgJ2ovpo~!9yq)j--$>vk( zcCK-WzaaLxk$~gElel@-TJA`zt!>ZQu|NUsvUdG_yYo11UWz+kLvC`_wsAYr_yeig zSd(8A>M_qqjJ_#79YmDxK8zG~C@^BLmP;nkoaS`FWHZhfw>O7hqYA3WCCs1kFynNv zonIaskxH)?Ow|JoQk<)Y@qOFyBc&E-Hp!Gwuf0m!;yU7VS+c4% zYtOT7v^ymhniumbzsSK`^$C1xjc7TaS=T+YTUQn&o(dodu0GDuhx0l7;}XqXKH%Mg zXAIa0mF?wxH5dRwlw(?(!?#ba<0{zRtpdOm23&{|IP_3dFIH{j4-uSL%o{LFoYOF% zEQ-+X9GCbvND??WHvhGUuuA?)-Cx<_vr|#r&lF$676}AO4)HrP9_L-yi&Nka@G6FVa)-xNs}Bk#r#iqW%SliZ_+z zQm^S9fgqreDkx_9!+5YXC%QC%+L8Iu0%@aa|r0!k$OEvZ>HGb{Q$ zohn!k5n@DAwCg(KK6tnfvbq4pow0*aNH9oG45ZUA|Af1T7FRr{R&FjGHi(2^m-e~j z?3GV@!H_^I0NLH>P5XvpPdl2IE>H#i1c*}!ba&Sg*%yyl#s9s3v@pm!j6M=!*_Bw3 zx#7=9EZJ*iqDs(C#3YK|Ym4BS9emI!?RO<%el;hUy>(Z)JK_FwKN3{`Xww##ns{3< zW{??{9{eZPVZF|~8~ngsyIf8+pPN^Prxb*hGQje4zx3RZBd6fO`c&%5OqWYft_gLZ)% zehCG|tt1mZAE3a~&(^`g z(oQIvd-;X8&0@AV0gys)5$M!_C51jZt)!y-FCLWOuI&FX^l&-}opOkcA+=3H4~XJU zuXI2$TwMhcA-5(U9uuLlE8E}h7+UsDJi?z+>6XjX*FlLM&=|*oXXS@V(+0{7NF<>J zLIN)BkTp>@9?Z2egz=COzX?(^KMxDl*vMmQ2b+(4$T?({I=q^TDCXI-GuL(P2BH9) zVtE4)wm}&)RcaBm(a+It0hHqLL3_-c85dEJL}8fP9!L_`&{83Qwk{qaY=I2=SS(zg zwZ>>&F}8CxtaJ5E;jpmnfv&iM7R!|BL9PF-bD&G$1wh$#C{2Y-Y&P_Kz$yW!{S>n; ztclqwk_WpMgCQP13u4a$E4c+=v(CFrvHeMxG{*ltsiq>M5Q~kzJPAxn`Hj)g4}kQ5 z9#?WU2w-9wn!*QpQsG1p!F1l3UZ-8yZ<_^2-eR+mSldLRk#GJeL)UBYAC-h&w--b|718bN67{X7o@XrYrZlO$i%{%TN!nAp|&kcykr1uA3TL zZ762WmhW2G5B?b_xLzdV@}5K16P{~m(!h>a8s2ln*9*gc={v8lOR&|L>r#U#*LjnZ zz_-Z@z~^-vM-SA3$^Dt0P!| zo4)P;-vXbt;r!LraQ=(e2OHw)Pr#?GNKQbtXy>Oadx`^Ke0%JzRh@zt(rm-}ZvM9| zwoIW6JOBu7A=2-wXpX{80|1JeI}`7zNFL}qe;A()h?7@Wc$)!z5g?U~qZW={DmGyg zMMxz?i9yB812jdHN=HI~qK7J1LGgr>K;nrN1=B#z1O9vKLf2po=;EpsUBr7dN%g0q zg6~TNl20t8O;m|0zd^r#XwYfp)k?%S1u>h*0}?LsxY-w*dMEVelK&RzG@2Vr?`eD7}Cr2vP9O@Z*MQd0&6U|{@R|SC*gG9p6i0(bO#iU zDVl$=R@}sP5$$2g-W?YL)SG!e#0IE@=--+VSvIkdfd<*ba1O(rmiy8o&F%Goh+5vDyD_&VP z8T`oHNwgmWQcZh`07j>xT`yJ$%BnU0jnKw`I_kxiZk%Fk%28%g8{E-g@$sxX0-VWN zkk09F8gYtFcA(sW^!D!5ne2B0`v-7G(8wDU|IayT%1tUmuS7WfJ?VmQf>i%C$MefK z#|+VKEOE4mFj~COG+e>nn#!qCudQ*3OKiLu?Rsm>S+*4_)J-^6FWmlCAa&Ic_D2Q8 zvibR(YS2F1L@~>Vt(?j+gC-f^4LW2Zye+}ucirqJ%*e?*9=D*IT!M9{^sHv1DhpgG z6DrPKDF(E-QVY#Swr_~z24(yR?SV7I&JUny4?8~u3!nRn7 zs_aU~1dKaW3ciAc-Vh;|=C63%PtJkz5rQ%L8PX)2#r90B(Q?vTSi&>BO$>TaQ@+2N zL5d;Ie%Ld$99Z67E_IgP0ZZzm%^;>?DfO#EZ8>V{wX0{(+{7|DZv5d*kf4OOHZN@f z&TLqN_@yMwn{txbTVEd2;e3}$U*7e_&z`c)Vf(kDX&>v9dVx6E?zlTn#v-y8e3BZH zejN_BjTzg$R7-Kv<3$?(@+wn8ip;92|s+f;>Q6^-$2ouG~sG~4vPMG^~~ev|7K0#%F_xDt3|cIakOyHS6bGYTW7|zWiU@%edO}hkgJDY z)dKigWk5Rx!D#p2j_WymT&1okKw)X_4g(VU8=)_z9!ikHvFhwyFs?^9ps<=gwxPP15gxylTrldbs~Jrux?HV^5h^D$_YZiR#@2; zdh)Ri-eu5Y(fsWn9ZB#QxpT-QjmlU>V9l%7?xm>7>Th3^njSKSs-fRYwJTJ=6)PjUCsHEUbuhvMynl9-m~ z?`q~FR)xetB2W6zT%x$i`x!{-M#v@#huwp#+x1t)(;T3GEltKHE-4oPO_3!;_T6rC zUQ6@R1K3>`p!Cb&Bm@+mb54uD^OGxz5CIB|Dn(UoQ=yHy_Htw0+|WypeRr&tAzQYx zX;eg(I!U`_mPHl7D|C_=CPxVY-kf64f{LJH>n4vArw+1rcu_zC%k3#X$i&0;w-;Xk z0wu;)U3**;Z9Pup8L^!o2c;*>x1#4qM~=O4K8*e7>NDCU%o#8p_G9n(P_)eE9pelY zo(od&0=#+xWdU5floq~bO^?aAZvQFX5UZiL!#pt-Bzt$&>nKUC=5~re88pvI08TXr zdbhHE-`ytC)w=+ZmV=5!APyAhtwk@g zd$j54?3k;y9oAH^#495|Zr`L{6xhO*4_5?d!f3WM8KCs86>FAew}|$t7<{v*v|l2{Uxa^LMePqawQK$QE^cFTc*30^i;T+S4tJ_LbjBjeK+_jH8Z+?;BtT!EA3&J%C-U0dmw#AMaJuq_OJ3>ge znnBW>+4S3h6Zvp-IpVRwX;I8(^fQSlP>xu%;Q){}KgTCes$MB@)Bwu7*;JinP6)si z2}OEL2Gi3E@T_Ii;i*zs9kL^OYV@6*RW*wkw(}Je>=FJw#Zk-OJ=YbO0nmMtWm_0K zH+%2#H?hTc?M{PZp34B3_zA#ug;+e^&GH(s+k8-0B<%;P#+_vK+8YIthoQYd2g!38 z(O@kC8VUyYH-u1k6Qf<>`z20Eh175Whg2l6`p;+RstIfmOOJNN^eOZeefBx!vD+h* zE0d?2HKW%6nP???dys29X6g71J^ks`_D|tXiov9~?vv#qpWvmS<=J13#S@3|hq!aI z?YzOXeC^+NItTAAU77=JGXPhBqPzu6PA2|{@b}|@tE)>RyBE?LP&j)9TCXekzLT?_ z?QHo{SHKt=;_m`R1vN!En;E0>5!6Sj=s&YyMSl38inA%$^g@eJ-u|Y*F<8|O^5OEs2V6~r~!`L zSzPmaI~MJK@yk71`1^NMABjO$)54}4vTlr!4{4x^`dw6`L9HeqdT`{>Vg{4|qBz@F zV^9O}9l$Dl@vMId{drE%-&+b_n7DiRaKfOPkPCKdrLrwov*Du%MqH#Z41LBNhz%@I8uU?oaB_S!E!Gr(58 zRG`Fkawg_NJM+Vi{a$|S@Yi~q!oBuGKODKV-fppq^@_EVYhqt1c)B#cK^Q*TU|fNw zC7~W{V@62@`M|QU!nkKWfcoSbuw42C;?x9P2HR{eGGp`P4ga7J+WsRf1hX{ZW2UXF z5xNDGIWC<=@nMMbRJv@TDEAqS(hMh)| z>Tlt~1?PN@E_tUx>CtPoJ>?GKj&<|NunHSKzzISl2r-v9iP0@k{n7?=Wj*76CKyiZ zH;dz~3hzO`O}RowOP&V zxqOr0#Ic-Bhg;DCC8P>W3}_HNh8{z!qZhJo#rB1Bd>rjYw*eAzfvS|W8h}tkFyBUH zW(Vc^B%Ro#?!>w6Jytx0ftP9zgMe4D_7pBQkWZBFxA~p3d*3Ot-aA?>89;STcr|Rk z6PjE;7wVU7)4=8268g(Sp0a9k8?nOlx#b45;%+|E!U~{0;5{-ZyH3@{3DL~8VOsHoj zo^}p>hceK@%xd^Z14)vGe1m*Y*eLtMUCP%3kuMj9Mlcnm$P?FoPS`EasyA-Zf$5=@XocW?+i5fpsvZ*L?xI9S^+;M7^qV-#=gz`uR& z{4kM+L5IT}or3I-1uIeh4m{y^+FL0ol;W-AM+x#okdXAxzk~(P9NM_?wOwnQ^;dfY zrz)iFDI}>XX`3vClT@;#88&(0R4W{nTKnF$Ja2oh@LZskM$Wd;M#f|$f`0AXd;_*X OtgZH0-rwmF{r>@&i{;=+5kxOP-L`nU}@4waKb05F&F^tW#O@1OWU{?SD18lajbPoRVYj2l#);g)L z;npRwXAcBySg`#M2!5JuT@`F;@j~5F4mgof@Vne z4yj<0-hs3q0cHZp3XIt2!9st$T_WFKi2A`RLTq-FSDg&%fHF1fdC;E?#8NB}KYmOe_*4e+?dLA+CveTfG=Ars z$Wt<3mBgprd!QoI&i7QNIA5B8+_Ovtbx^!7?g4Ed1%C33ga665lUV;DxQ1I1(*>O zK^~timbU9Owbd41U9RGDl$4+X^x~dK*T(rlA|%!>JWZ6Dl!y|R$2Tf4Rw-~Fl8Hf< z_G1xiQ<%psMdY~uaU$bw?}HnRsDO2XT`7>&A%PU%Q83R;8W+OlSqDg(-vjs-UJ<2@ z?_$OBn}522+qSwOPwBXI_|0$U*EmPRPt94;e!kMpd%0Y1jG+TR;F~D-PYqJ@=EHm< z!O%}dhvK2@Ql^BKU|skz0e>St?iGo zh`8V16ajX=#s+|P8CF6i4hvi&;|*3i^_97f!+zV90eZ~xJiHc^3G%^`~+(iUuh7@e>Fa?>@^v#6ND?L;rcKyf%{nw%O$hK z8TR5V4NJCOK3?^y zq5)AkE*BHgXLp8EijrSyggc6Lemi$N88WQUIj6{{j*qKHKtdK$5hJ2@kR%ux6LPei zEkKmAA_ZCqRSM$S&=gtTq|vZm*--o*%EM!h%Q4bUQn={|4;u8rcU=d_Nf2V`>t!@O zi!yuU7iz!}Ap@b8&No4uDQLoq%diPOQcepJ0^-;LCN2u1$h3g6ApHh({u&xNA_2q<>YzX9=(A_A zVKBZ%j-_Xfe{x_Zp~snDY57h@``;mI>L4{jq;jlHK;j2z-wVy$&Q}&?dcMRY6uFxJ zIZ+JKs14_Ai;R@On=0*Cd-f&odrghAsYS0LaY6X(c^wKSY5-n-$hQP?6qYo^Uqo}| z@P0@^Qu&1|-&ztkrHLXoI(3X{B^n$eKfV&O8z#L$(k+m$^7v|~l6Q;|VySV8GdOOF zx$r$_^s^E68$40uYCuB*=3%liYWNunpR6!`Si!QEt((}Oj`xbbG6Yv!g9JxW zl$S2`It9BV_9i7Al94Ah?smamQu#woQj6#s5DWjTIWm>3g;at)MtDA2)O_hiDX8}p zObJbQhX=bQXi5H*6+HqYs?dO!7xf~wMsM0ojx_V?Lm$7Z$@zM5%vrFk9&eAhYk>s! z01|UrAX2YqdCXKIxb~s&?q7u#R(E95^(f@X1Dq+}irG!SrVh;a_ofGDjYU`YuK4S> zpfNV_qh(odX6om{U+HDfv7O(hrk*V|D8x=Ui4?vV?IDgOG_w5NY~Bx?*=sWo-yN1~ z&yY%#>FZK*)M>F;yQ$gY%VV>p*5ZKhVZ;FK0i#X&>XRwIG%r~-qA;T@po|@bnP{YG7kDh0jT%#B`!z&sJyU-P9 zYp*om4-vJgHX~#13HYMWP!{pzQ`zacJ(5tm3oJE4=-=B(O_}V#^JosoRe^BrwNw^9z76p}wAOt+H2V0}JbrV^sYH9uZEgZJ zMSb!$f6CDZ)g@VJ5NYf%i#gQkL z)R@2bSrY5DIP-lcJ>abTWjEbmv#>p)%$s;r0@!=N>oH1}E85^&Wz5w=Q_dAFU2jCn z+=+js)^k7ZDz@$>mB(p7uQpmMHsB^N_~6M1%Ta@zOKGp>w$1CpeUy%wTv#)_%r?R%6FjR-n305g{Mmx5h1 zekI_FS0Kx`dlf8tCxeL(n{f{#>>`xva1HupQYQ{QS^xEjXxQW*F`+=)-3sF+mYYW{ zB@Bv$Xd+Sf=s^l1qCUhnG_f?m?CB9va@`ZACF$U?n$N%YUyMViZg8@;5+z2A0$YaH zDRc$gdPF3QX~l38-_mpWql@q+J<3QjXjfw>3VYHGt{ueQ=%F#`%(eo7zY}-w8;Frj z8DZL=Zud&iAFkBA38pULhb8>mIk)8qF~r55aV$RCU!YVD+7>Q0>~-zUBmaj@+49Je zSp*t<+YY8u=n>EzD?Ei}AjVw#*AemD?#m_Si?d5=CQpSDlJJ6KI2Fm{PwTtA zDw{(whf@MW;R)tiWxo^;P&gxxVqW$sH7Oe0MuTI=)ELyuH>&-EQb^U5&k_z_)?t68 zzU|~i9HZ@!XI4m`Z0}Jg>Y(1$knnF85DVQJ5;p4kMrCMo?)?!~|82@n zKDy6YowxW-9XqWPybiic8e(AwDm1fnKc>*SLnB^!^ERy;3ez6H+%6G(a6Qu%Cap!P zspR4-Lu-avShs{jori(fF=>?tH4P~ll!9E=ZffZQueE1QUdc)m8(2o7o1x%?7pc!k z-=5L6{BFf9fZex29+1gvD31|iE2D9kqLW@C^hZO-!j&o%*P0B>q*)2rhGkXcvU$LL* z)CL2_?}NCRfVZ96!Rd=Dgs?kW@8^~zU!pxye&PKkNX;%w4q9i&VV?59l#r35RrkaV zBrWGpT)8N8hMzZodc2w$nsj%3*0zQriX8w_yd; zp(Frt+(L@x`l!M^k&~A&#~fg)(f3tv&GrGgDp2@GMzu3w(k)+MuH^ z8Zh?L*a0bX#{Fp>sWGV?!f}gBy@S0O;TJ~dt<_n|$G33LuIJ7@u(|oSWaEoyZC^Z( z+^Q9H7VaB-H^BbaeCoxHgO}dGc5!>kNNe3snW&+e?!C(b`T`OGy1Rv+=vX>cn+&#; zn(j}lZ>-;<_-yH=9!Jm)A{hxKJR&0xp>zEU9kG~S`~406>o`3E4>nrvXXG!Rl-_iB zDC|zzvfe2*HCX|4iwvPcJlP?%cRou-`PLEq2V8P`(+GV?97}HMS2W|hjt*ul$xsq* zxA@}XaVM&zeOGXT8s5Fyz)W+bq2{cpV&3v#q+gug0SH%xa=n2G)t^$d5GF1C!+o$@ z>c(1#eJeVW{c?^MTvlB*6}SZ+thibXE|@b_B)Jl}a)QTvQ_2;ZLfH?c*W`r`CE2s; z;h!MI0`|rOlZSM`PAX0cFV9$YB7lAJa%|2QB~c7k`^QNY{yI@PzkO74ZvDyo_jGu> zx?z(=ju1NIDiL+dc|*6((xL;~RFbDiTTqBbEbg&4x{d6RDRNlDT0suE5dJ3C(@=VU z+lYb7{u_XVLz-Y~o&CK{v z@mr$x05p;vepoS~BVhlNgcbV>+RH<2C`^p|j !`N4t7R1)62jV0_a+UEx!gw6DG z+hPZd>o<&AZ|*X-TZKtOB*Y@$V$kA?t{-Hz$icVa7@n+pXewZ-(BC5LCIi1nwEVof z73TcW1fS?dX?6PB@!=c6zqfj@Ur%)G&UoE{f&LLNkuxec8R#3Qs1yiE!Ym`i4y2R^ z-(}B~lxM1~dZ#Z*u6se89O{)9lM4xe&cg>=&|~Pw<+tAzXDg1UO?izSg`q?ZV97!& z*-lyPiO=FwpBP@k2`_X$(P%S?QA;nUqctCYX}3@z{%m8dftRMw|E$oa zuB$~?j_%T{mupn|AT4rZIOj*3QZo8V|B4JZi;giEM03Wwv)g%uzFe=IN(in4Im*s# zT$uelSzBw}{zj%KO&oh|xcxcKuC}k?}YBZ6{iyqJA zZmmOeeuPi8$o%=Wi)o?PR)72J%4bLO1mzD>cxe)S?DU;%dEvP^9c<7&;fyA2ZGeTV z`@(S<*ZBu~BJGJD*L9E{{Qlg;Juy8>jyvwHL37k)_xU0XLs4O@W-zh5^P+`aPmxx5n0a?3l=95`0)A=m2kesP`s%>Opv; z0q^<0z5By{ulEbPzy6heCWbc+OVF%&Zo|Dv=qV4-fOw#JKW%5j&!68c**o!e{Y5UC zpark%MY-e!!^OYq5Ji}#OZuu(!93TyVSiw3?<#SIEcq9t5fr>_acr#h%q^3kG75|I zd?>$cM!WY;x2(7-_6Zf$&$WmbcrADYUN31NEbSVrsoYCXRKdQYqJ=qZN#P3{%GN<1 z*GC_%D7tdc8sG5q;fd>K{Nn6zkyU^x8?Y-=hks$a6ywdwZwkWL20!2^?=86(AzB@7Ip&4p(UIGI44(_6qcVphLD0R198=4m~hDK6T#khJoEH%b#P zn$T+6?sqWOnM5HkTv$xqdapLHTgG4-KPtd>JN|N}sfJv1oh*Al8gw#I5>-uXnqiS%;C_F^Z(?w z6($Ik5?hD)YeJa*3(7|_{i~yawGWV2Fi<7JX$c+UVjJEc7UX7#pY-I_{1>Q~R9{j6 zVtGs>xWIFClq>`}qWN0##wvZftbO#zQo^Dz+d_}>)w}Y(4zcZAg_Zy9M0B(S(spX2 zaQI*6{cvH9Mfm~ooIp4~geiSemqAm;)M!PAXS0{*H;@<^VxRAn5dkB;rzmUHIrrB= z>yw^?F^m<1`}|h5vfm<$8j7d9BpOC~JyjMz-J~>9crp<~GF%MPwNQzduLPZI(V_NA z*-uKTSgw&|3XL~rSh;J(5Q7zoyPUaXAr6F~b-FUaM;8OtIZk`qaSEh4I4+ywK^dL6 z93u9O1QejqW5~Q)2Y=~m45R?e3qQN#F`z?LM~RX=?L{FEK@g^tQVq{n{ruTm9iCAe zMdta2@6NR$I?)A>3$E3i@!f^wRZ_vii>dBGx3E?|z#;*E6~am7jL{q!PaXf|&f7Lw zo9wWdby%^^6~@h6 zw|@X8$+QE|eA;ATZ*#l(;m@c>apa2{Zmd#{1n)-<@HZC3<%C`(FRfg-^;#8xN&|xT zT%cN$B}sIXbeyXh$;FhG^gj4s*YN%duD`tSVP$*|^@~)x6r=$2#7_I35w#=il1n~R z^juxlg~uBZF~Q^2*R;3cZhjYdOZ|-+r>PQ>@cic5W!>$>JvGKbC@I47`n;B~MS?0Z zp>^$CmI$zcALxx1%=KfdzwiN)$oI@HZP!dt<;F#7FrR`^T5#av@nqc8L?JPs6ju)| zY*dL_ZH+5Xn(w?q2pg_1gYCDN+JE`ScgX>EKU zsXj<{f4CHOrQCZD9wTV8P*nBm>J2|hJaG2dN2u-{rLt_v{-L39yo491MB*Y-DZV$l z*dF*6EN$lDz4hmsU)en$v4|CT?PLsZ5aXR8UCHFTp~NjE6c^>VEPs~FRT7pXyjx$K zxRoI%?EKviw5k-1SX@OVUpTWm@)vh7>Xa1DDX{k)oV)fS53g6Mbp|>c;Mo<+;v+^E zJ&jp@jiMK4Zn0d~Y%-1^s!X@rO0_{;Pqn>Mt1O0h47kV0k;eru-cd>Z1#ukuGVWh$ zArT%^g4s}&jIu2}^8sF${6v+-gE*!yzcQ24c=$a$DSd`4>7L;ol9Ba_G)` zh-xI<^@4vNyUeCs5Mph?pM6?>=OuT~mi7xeFI;#2Y#(%NHFG+*yCPS5-c1X1D?AzV z>06L~TcMkPIKSDJsRM8Mvvt$`>xEZBc0f0mLd#b51YrI*W^So?gQhq%XX`W+GIo%)5Pq zaDS_#YYxCo3uUe)Ywz+?1p

Bp`+y3gRAV%-)f|1!!(bG)O%@5Kj*T4gXZ(cB0aQtXPosW)hIlnDsygyo$2rFnT zTRNATzaGX|lDrT+`@}QVazk-bi=vtBx4 zyMST$xevh%Axtjam2=K%YqzEzSM)5{ zqm$GasAxniv^BXuVI6c-k+GAyA={^G)fWtWA^8JOw?&RAp#b)xj)kJhEa8~g)cIaU zN+^Ihc;Zlo+oIP-;{EYE>gKExVXumJ!{OnN7Z{S{R*zwqcTi>_1$;4Q)h_1zdNjVP zcIi(YbP-K?dd*5395%&xCUcaOY6u7ue=ORT2SMZ!))qnt-zhyb)JSF_j6V>&REAl* zWHh@6y=~Fxz&Z_X{D{0Wb~331glua)c$SnF^vmf3`wmTl><%%7poKyT)O(vu4Z{Z9C%dnp`G7R?8mzrkB6a1^DQ6D{tKE6yny{g&9vea68DVQrx0wD!aPhk~cb z%r5xJ$CVe(#8WH%R~jt$_I&iIDW}uL>!-4IxZ%{ zjgNnoTM_*>mIG8J$q;U<$emm~N}2fp{JzLZx%fDIzMu!45(gm39P9r;F)1v7{9kQ7 z*TxV zux=G`{N?cD+52}JrIPB=^yf6-J_5nLz1P9R_+u|DPCqq>1sBPYRha-3wIbv_oG5hd z=Sr_8nFp)jh1<9w$d`qxB`Tv(>K*~VEZ~IhGvWi&g&DsJt-=HMRk9+!QCWhjG_Skg-`tSD%N(yom>>q=8@f=1SZCM)9EYO<97Aw2xU*Qi9O#o#UAa$M^pP4@6P5(~n%VjY4_UgfAlY*j z7lrz6T;Je%VeI1x%dLP^DIimuC60SCiF7P<6dP z{_~OP<)FWEnKYabJf+C33>Bx5&p+4RGIZnHYDnIx6X8$5zpW}UP@3TA*c{6LKSIdN z9S*h8rZ8;@g-ck5kQo$wXW8#3j|}AqP=ax3GFfpvBX)7~th$s86!Nl83A8NwnQQ7m zcxh!Qjh3JxEmk_!?wU{e>cWGt!ew@>5&%h|A*_zdS-qTJg(;;pJd{}Mg_wA5c$O5G>PeE_S%Uq$h6vr1PKm#lf-b4x!& zdeTALgYliI&!2P2e7`-l@$-iXi=mZP({xsuSPrG}_;kBrcHESl^2hK#Qs7qNnv`&J=j+{ZZH8MU zIS}I0khZqNVs*gDEGS_Vn87JtyrCPh;YZFFQg_+n0c5L#O_HsTuD{>Js5j_I!28d& z&fK4rec%?IPAOH%C0_5W+bYmlAYd!flL$hu$iJ+YKQ5lqLmTxFtd))CL9XhAm`{_% zL$#nMw{r0qA%hGFORDCYo;ZH~_KW_WXY2N$G7s^}XWY`t00{#O`!(WJs>U<4a}=V7 zzo@k#%>w~3h6CQ4zcE@n#8d3j1K3%bRfoUGNV&WvZoM*LA@guK=ju=8!xbs2S@14W z+!XY}8y)o{kILR=zuiCYCPCvTw#kl?Ru2t~NzUuuuJk9iEtQDpKm8t;#8u^8X0)cP zK=d~C;2`m+Z+?G8B4H&>kA|%U?Y`^oE7P23Yp$32xNw3;W88%ig(9i~{l>-M({Oyp zsjjYPPkGHUeYwE<_J8g78thBOMZPb~k+wA4bc14V(LCP1#a|Htl6a!R)3sZcS|lM? zJ$~rwsD@h0hVZ{5XF4CuFT@}J)+wY#yd%ZML0ioUGW6@_?6JP76(n~pogH+FHU@?^RSkRB7I z(@=8zKMThm-^_X6$EMOL&!Qg8;_YU5Z~2tgBdtNdmXo@b`>(XFRWewxX@zI?d%x~l znXLRc`5jS6E4?_256ry&+*_&^^vsrA+5mP5>U8AO6CSB7*Vx~=UB3H>tAOWop;9{$ zu^(d5(#Ff7o{Qw0XpFABpu1sf$|QDgpE7)Yr)vAw;!ZOpf8~J)lAUdXDwYuTku$6$ z8lz$mMQ|%0AD^ii%n&EpbyydFucScnmQ_`<_W0{#OZVSk$_p+V%BaDE)p*o2aGky8 zr>sBv&kD{a8I!jk!)07Jo2nOzPgvT^aAX5wsOK0@>Jayg zN7v7tOPsU&jS0$IAGX|ThV5a`1WMg^>vCoB&KkcE2OsPCXK7Zx&0V-6o0JR*%6&sr ze8hNr%ljf7Z`V}&5HZnx^@=>nCJR;S)8~)g;9D&ePg4_e;ob8Uv&%7_qj@K(4mE$F$=ewr!bm1Lctb6|(`^33uYHc0YTV;gS^*p- z`&-X8b3pATCBe0K8d@#!dbVTKqW3g~TmxPCi&U$CcWcez*|8FFJL1=^G#ImYb^j;D^9=*-1GUzUD@oluEvz;7a1zKW4EM}3-;?mbn{ zGQJFLP!zZzJ(MChCH~u_7#4-^##`9E37l_PmV1`DgHGSU^`i2^)c$m?n0c3!$8ba) z?%o|zKG|f^FT1)MH@JDRs_L^M0VedB67*sG!W#TL6mPrG(kbn?ZZ8V|K97xrHP-Ii z^iYk~Mj@Y$g)Rj)H&q39OJxM7c%4$>!wfMi+Q$oiP7O611S_a$ zf!-R%&;4QFi>ivD;$z|S8xHsJe=36%_E;iONFI_vMvPL}?>tAtFHjgB)p)d^|IWIl zYPTSP>E*D-&j_V zw(LQam?c3|#ay+~#V@-#v-?$GMtTveGF(7dM#ugC7Cd|w1=SygbnZ-SR^*=sbqjHJ zKef~AEF{831siY&?iM{TA}fAs+{*aIe+N!C+itfhuwsS( E4@T1M_5c6? diff --git a/src/main/webapp/resources/images/fav/apple-touch-icon.png b/src/main/webapp/resources/images/fav/apple-touch-icon.png index 237e5ebbb0d41630db3daabb9e8ccc3024f6970f..e0887b9383490875b9c7b89d7982c4e0e2231416 100644 GIT binary patch delta 3041 zcmZ{mcR1UP7RS@7s(R^Aw6DI@ZfozNRimheqIkV#?HQvrYW8Q<9wAn(mWI-zMx?eH z6(eF3gxC@!Mq4#@=Kud;>jDY%@V@i@o9 z0KGm}C5*S$XoSU7+FLw4X1O+hX9{z6da}1XvsCqQ`2F8q77x12@6Y7D-yCc^J))5Z z+J=G*dhBI@g|gJ+ed^BiV4sWPYWvfbn*jPC^+$Z%l4hI{}l*nN7U_&ZY1Wd=E~24)8pfz0R6os z+;rxagZJKe>oayd3th8?r%Z6tMq%Qj)a>}rh3g5L{esk2jA&#jdi2l zfG0bX{k=}|sMn%X={_6XKTl`}{hn&`B?&8ag*#aEWQylTZ~at;@6Ob~d`aS5aa^yn z0?JDGXm@8U+Gae~{*biVZ6h%pVm$W2diGP;Xt??2Q2Pc7dAPAUR}?!QV@I8zIHGJ$ zXZqm_KTvQ(s||2sYuWzl-1d0S`tKTGBHnQ*=oQLJWOp9h@AZs|ADe`FY>lE8e`L&j z3Y$oD-5lz`7e*cL?hL-u!Q}++E#rC}WDnMs7Aw-%yAWF=zqcp)h;8Njg!#43s;!Z( zwYG|-s;s@0*@*<_{Z;&0`;VDC;CO$x$4;8IwT^};)3!H8!pxQtxrb!pEbuvEEZTM| z&6_$qI^YXlD2JgTPe%Zg0UvE_k{j-0AjbRod};DhW#;bU)YymD^QB2>50&-K>Y4n| z+0WqzYm4Om=E>AI)VT>lbIDS5_TlE*fbR=3rj^iKy3tcdX!%B&9T|(V#(#;LguU6u zqDj3CtBu9BzG`33PyGU53)XrDilP7O`MF~7(bm;qSfpdVp{U4_{DT<;x}v20Ox4JL zij+rGXEVO_OUpvv;D)9uJ)5qi#-;wR;WXU!4mN`~0vA4k+Me~X@OEnw7h zurkGIQBb!zTh=qoT|MF*nEQ49uNq^wrw}3L8&;~Zi9T6riVV}w_uVA#KxQ3D@B(h`!)i9u2VE$n}@z@ zJ*DIA27qLlv1M%Vy)0%|g{*^4W&8U!PyR4Zvp%?@EzIi*ABjd146IKCk>8%7o$G26 z4GtdLi{R9|Z2IC7GIO(a^pj%JD|%z)Ak(f3g0Y&g87pT)_+lJtyxj2{LE63H=<^-h z!TW|dRa%^0`o|ihLI`8wt~s_n9nJ23)3`$r;Agby`M12G#}XZA0p6Z-918KYyvZ(7 zhaS6v7+}4aVKiee@|haacAr2s@JcLKT3s zYCsUzmDtS#>+UV6-w;;b!lN?wSQQbJ`?gQ7!3g5i@Z;;EH*Ks6{LlKieGt!U%1)jV-ojGU3^UG+wU1W+rr5 zRtkkqdF^E2+&p8c!yp-n4=oq2G>~$I{kT1M@I3CHcu`?n7{+AQuvC~0sZD$ADUB@a zWWSAOo>9aRJD?t5xL&_hs>$yxgg$<9x{$y)-mxPNiUrjRA z4M|Ur-wOKY;fCAovbo%}BvMgZ#)BhEryh#iX|u0x_l4w$XqKbvdg;y(0axVF9|Ka!zjmqTui0%LpTBjJM>Nz zkW8{dDl{ethwIC5#2&81G>av#G8QXFg+Cvj+{*?y`J&nm4HC}X51>iE{R+A`;n(aM z_3b$MwUdAO6@2A69{pNnS)y=d*s1ayOvCHvU<^fpJ%RHPsRBgnk;J9?BDP%aDdU%p z^I)iQ^m$eQv?Y|Jb`5Qb?)?OIJH95wor#Aj_z z=ZoD0#OPXYaNKNdAAi->;#NC+PiY|9>MW1}nJZNvZNyPft`ILuctK1IYr2fz=*UO| zU@bCg;QEVrPu)rtm4y}2tIIF0CTA^M-K1E`6?re0TE>-|YrN!deoJ3Uk8NjL;vQSJ zc!TfSR3b06OId5R>3>m}Ue5G+CmwK(DSSmwqJ;LE z3=={;A_Oo7Sxkt(a&qmXxbzcp-s6eDDvPE_h!U^K9X1jYRN|9$T+ZLa*%#1==A)`6 zm~BYoSsldTZJ=NYIdt5u>-VeUa;;GRrHO2}wy6BgGh!@g> zBrN}Yzl3DHlNJJVSPB!f6Q4vG_A8}peeo8pc3w~lnRj8H+Vx6^AkrFM{ug&DcXitPOQbb%t!rVbd60~pZ5=vd4y{I-M=3YY z`!0fpRiUvyZ*6-S$bg`sRe(U21Q>rMOzbi-QFA8;2A8eM;&HeRw!HSS zRoZt&@H!I&<_q*YwA^FK+-295k3pUdrewZMP`*_yR?X6IHBpP;f5GL&U6GD$x(kID zllS3P#$d-DeTN=y%B}NfZ}>;LURQ14Uu%6=zM##0ZpReHZCaRO+$M^z``{O?takVR-AvMXw&}7UD9s zw*Bg`O4ND#J8$NdQ;8i9xiC_Iz~v&yTFRubdVpI_IRPk}_qp>USzlV@1HM`%)BBYh zWd5OAukRxuvx-z!heD=J)!zj;!LiE|rD-`?1&=c3G^ðB&aPVu(kE@NdCEO2eF~ zitjV`gz6>pfyJ^1r!o<|l4m&2#aBSn>ZL}7t6waR_FGE_^0l~j@7A{9zMBS1_RT9Y z{iov1RsaM}j@SDw@b4E}z(?aBAA1`ghrbo=-a4EUBqbrKASxj(Dk*IwDX%E|=)8p` hBo!qjvK+dJ|Lah6_q2C$4EX;B;v+9(81)a&e*yCqR~!HU delta 3300 zcmV)*}7~>j{G=BkeQb$4nuFf3k00009a7bBm000id000id0mpBsWB>pFv`|b` zMgRZ*0CboQeVrwbssL`35`LZ}mahP5kpg*}0CJW9cbO!OsUU`=B8jFWm98a|t^|6V z9)zMMk*yhmp(Kl_0Ct!Mdz>1ZvjA(87J#22m#`C^v;m;EAAgy#37xeikgWOq{r>*{ z^Z5Fx&fC}N@8j?D%j4_B-RAA|_W);*Ig_t3j;qAx>>Qc0XRg9troDT&$-3I)lf%+a zp0|g*&P18BbFsy;)Zm@U*eaW|HlDRnsJ*A*UHt$63)M+PK~#9!?VZ_nqc{_QjhEOM z<2~*$`DX%Vhkt2f%$oT9-|J^bLXtxgVBnnA$wgmvzAk^1N~OX*8vR*+R`*Nb2M8?( z!tkJNOBk03PQM~^QBG$)5f{OE5}Pc<2a;75Ni1tL2bk{?o!?O=xHhZ5V7 zLZkajUoQo2EW-@V5ADt#8H^vBKKr4V$740rpno5HKka~^tLK+q`f;`*Xrdnxwb)BL za%jS%*hfD?wgO!Vbsw9)ksUGggHZRdsV5mAbRKNrN}!LI0+Agh^~5OGN3rg5@9bCs z(9A`VFzehqJ62h!6KFH+dwho|_`8xCEI^n`OVnb$6{zGD7`q)d)Dbn% zFO?PS`uEFA8BzKiSTE6opwN#{4tt=a@R;UVkjMr-VI*pK)BmpZx;}p`J|Kvf@yjb+(@V zir3%K`u+QN6*pT%=rG#0WPhsH2z5doL!WTaV5nj0{jDrR6fY~SpHk|C`vEo%{59Qb zd2xrO2B7m$Z?_OM66)%0d5P$Ah zpg7Q63Z#DEN9|CT)KT$NX2*a)U0_!ofv{hpf<_2To#02!M-d@Uth<5x` z{(~^b3CYTy$CvMhnk)!dJPF}bU1~7galBvA!ts2Y>1011hWdaFHAwB0#S1VR!7re+xP4xGQ_3yvD+NQC_<1vW+z5PSX8Heht>oqjIJ3^S(`2l=?Flp#Q{-@toa z5PRJ~>r%F;&jguqD}NQua0ae;DyS#>i}Qw!b|dZ9^NpJ#p{@ur^96VtagIBJbDfV{J;{gJ7(7yJ~d}U^A1ucpeEQG(dfAY@k;27ihAE1epON z>6&xMs}L+Tl4D;R90KM=_OT!{Lk6!jc4X?KcYl+!O9Tc;BPsW_*}-MW zp&n}uOjpTXUh%O7b%xCvt|a6)9}JZJqIGS0*g87(KGJ&Z09h{t>Qs=LHLj9SUe_>i z&TgJLIrY+&Q0H35)CU_3^#Mc8n!dt=H;@dt)x5RAoE%7(G@d#RwFLm(z%xOH&0K7# zn&K^RhkV<+Eq{$yoWmVW)OrPW35)~KRfvCBE2uz~(8*rowl@-@cxB$JXgvmB)N&7S ztQ`^g?0M^0Qn$$*pL$P-AQ`nOgR==abqwx>7+quNe1~k8IeyK8B2feHEC_CK{@NKg z@Q7P{hgS-nbf6|JGmiv|@Gy?Y9uQB-OsNkhkmcgXLVpYe2s-9&n|UO&G+~b!b0YYG za1r(42xYmEpj=qLfKKO^8S~cqSXQtE4+JnZxWOpq;lN&0D9g#Nxmx8joDuXfWo~^^ zOSr&A2t*BTFucEu%1NyOTvJVs<^U=f6xpqdush z8MC6LdibxPH50kkK&Tf*ZDBhc*a*(Cj~$P6+7xod2vbKp`Zn8OB!4x|da7`+SL%OZ z#}s}TE7aRr2r zT7L^@u)BRJ>I&TUKct2lB1o`1h1Lvk9OMCYT3}~qFf`bmzMiO32{efsejKsX2{w1@ z)(&V~s0KBtNP!JN3##0D?$94nSHwmY$*{R=SrwRVtD@5R5L_*4VzA4rj=m=2oGazU zBl0|!7agjZSFK_vc08aSyWj~?ah7PX$FsOknK8$YWt)E7=E$2G2(?*xr%$`iz<*%p z*q)}EMXAjh?d>nTr+-FGS%{u|U_!x8vCY5$GGL2-D5nZ^N)2Fh*oLiSZS%81fqgKG zOcH0ni_noBD7B&De|MI$5l2dFQ|`)+iXxOx{LO-DERkSCyG>huNMjsyPW`eIAEkge zWy9DI_qZ)TZgc!fshv-n0({6W$$!}6mhuPh=tS!Ivc~mroU(JZtJq_%os+IKBSFvS zY^@irwd?p9!Opl(ek*>2MC7Sevv!D5_H!QXYPeg;A4EI!@fU8>Bx|w6U`O1uHvEt{ z0<@`q<%2!nII1|;fY@8V4Le@2VV-_<7%fWK<%I{py){~xPO)%-)(_vfe1DKH+bVX< zJ!}4Z2xN>M`uuw@;q&2&0#Cmw_rmYO4xRe;?2h6smoWBXyD+u(?a)`j_e=M9Vx-DF zZrTrdSAo`Fe)dfVe82$I3HS2aFtqqmP0l|a&yem*2z$9~IDb^_&`ta2yYAVY!PFpHD+38$1-JBo3-e7VjrED{46F25Zayissawethcv&u+ZtCm#?P-#n z+aB9FhXnA`=(OL(SGg-5O`nve>}WnY6MmJym^R~wO?USlVZ6998u8=icKO|n ilGTJCg3OP)T-xydcjP~-U+ez~^|ZQt6Y0y7^9yf(IC$^? diff --git a/src/main/webapp/resources/images/fav/browserconfig.xml b/src/main/webapp/resources/images/fav/browserconfig.xml index 5811d97a1e0..b3930d0f047 100644 --- a/src/main/webapp/resources/images/fav/browserconfig.xml +++ b/src/main/webapp/resources/images/fav/browserconfig.xml @@ -2,10 +2,7 @@ - - - #da532c diff --git a/src/main/webapp/resources/images/fav/favicon-16x16.png b/src/main/webapp/resources/images/fav/favicon-16x16.png index 4670daa6b9411a16635e63e36386487b47d3c2a1..a82a4965636e52536fec78a15e7d49462f8ba341 100644 GIT binary patch delta 667 zcmZ3LqiJ#!!HIP{epp^)PRBERRRNp)eHs(@q#(K0&Rd2 z2LgOTT>t<7eP|$13<9FVQM$*0(m+K~>PMqMBFAG@ zPsXYPEdv<@G#xB>EC!_hXsj}Xdo)}HsO5MRPytBG5ukdogZ4W(fJ_6iHI78716@C5 z*1-)x&)Am)`2{o7tIoPGS#G<7&noF{hxnLIupf>*HtBY7=T5!%jGwfgR0wALsJZz} zQ_+3e$+Sx<|HMkZD_a#lU25_%fTjBS0Y`C_X&gYa7?Zr+T^Kr8Wj%l#&H|6fVjyh^ z!i>kKoUH~j*h@TpUD+Qq^9ous2Tl}o0SYyHx;Tc^OI%J)U|@1nOA`xYQ#o+#+<}9g zN1i`o6SCRE!Dv^<$QI|AqU&@>X;IP%CLX3oLOd%Rg1n?c8~okGLK_16+~XKK*RN1n zxMtBRw&$z0GYt$pva=4dZYgzGvTmN8&jE&mB>=dO43V8>=f9r zY1_sPTbYd+>m`jjW74w3(t!?AEpd$~Nl7e8wMs5Z1yT$~28Ncp2FALE#vz90R;HF# z2FBV3hE@g!whli%Q8eV{r(~v8;@03J>-ZI@K@wy`aDG}zd16s2LwR|*ohJdWzuo2nc$)V6 z{P_I*^7#7d^!NV${yCGcfxFBOeVrebub0NuG?}tOm9QFvqCc3j0CScpjjCy@!2A9E zQnkn1>+q$}+gY;20BMqSyUe!O;*`SD7JQ$8!O*?i<#)BmZGW-FNuIW!%Ggt(xFd?F zkk#L#$k%hN!{YDrN4Lms!Os?%u|blqMWeaH-{)4JxZv;ei@(qSZIjLB?5xz^T%@}) zkF4pUjYt3h010qNS#tmY4#NNd4#NS*Z>VGd005pzL_t&-(^ZZ|62veJ1Z5{q9A?@L z^D^WAAGQ$Z;D0`iq|#IaOeg~IEOLI9QKoW>#6BtfM_`LAQt1mt-t#K`fx^+eZV1Na zkuBqRw*p#}J~7;25G&I>?0-$zBgDyj)Y)c5C#jaa@hhxbowDNICk586^K2Vk?*8Z) yjWD|BL9;5NUf#DcPP~&*C#!;YPG?aZDq)g*TlKP=g zmE$q$5OE-RF;)*q9*NPaKL#`AftfdF(BC^5$eaIfe5TV3S=_S#UNRrnn;k~37~eM%6gz^ zjK=XOgwytif($wyt92q$=R~y1nKa|$U=8~d3_zZa(l`~N0}cRbmyPzDN72 z|NUQ_YIL;Ew<@k^(q~_z@6UuTg?O)i@Oiu7yn3d$FJADJweG5PkXYp6;m=}ZJ$*{X zf{n*&_U0_$|9gceVFKp~iA8A=%03Q9HB6Hw(wRQG7+sA}zchJ{mPyK{;7cWX_J>yo z&uf)CbjN0Ua7yCE2&cx1EJ@e=1Xk7j1kvn5e}!eqcYk*oneA!xW!WlXKdp;PC$(bl z4KJptxd-c=%E}^lvW0wKv+#yF!^Da593N(15o>TZFKBT&`|Y?zA+uo1?0+IXD^J`s zb~V5C|IU?#|K6^$D85_pPT~hQ>vX*brvjtJUS^9IG*9@uuB6-AP1)s2=GX81Cv1%x zZV7+*^GR!e=!E+8e@Bn2uUMt^@WttlYp3tqFMIq#{HB7{ZTb4lm^fhQs+PD$l%yn< zq*^5xr2;7iBLhQAT?1oXL*o!bb1PFzD+6O~14AnV16zlmo+uh}^HVa@DsgM@k#+nE r)F276Aviy+q&%@Gm7%=6TrV>(yEvIUu^?41zbJk7I~%FVg3S2<2HC6e delta 749 zcmVpF`A|$$ zMgRZ*0Ckunil-!ys{n760&=B!@ zdbP)8sJ{+Vv%TBoQK7kQ$JH5`v8m77mBiCvtiWij!dj)f?)CU0 zhozUv)_%0ejK9zWo3z5*)bF0%loCL)zA4u5rq$lU$EpRYTJWN{cx2cbpf z|EyHY>5fL1aS{373^f>~Tux4D^?JTuAJ^?6gpn&Q&knU-pVN8)W?xohmSY6WSG5|1 zPPgPeE*7oU$;os&eY>Xn>+mwm6u$s;1oS2Ek(A2^$ZP4yvRxe~G$=xY&4*BOD}*A- zt((%fj(=PraK{mLpc;d4H%BxMQob>McF-PjMLMDb2BXM=PP0EFq+Txcy^My3A4pgE8iL?Lyk87_MmGEMlv59`10X+79YTzy+tsGP}A5;;kh# zY<`s#F;*0X(~p16KBk7%L<(1mQfq&ok~Fz13VM=3e)>6;d}uSXILDaVNU{6x;uwb% fsQJVz?@VH!M*5K%Uy42fx$iBdQ?29#lQ#r!HBEnD diff --git a/src/main/webapp/resources/images/fav/favicon.ico b/src/main/webapp/resources/images/fav/favicon.ico index 169ee68dae265f06cf1b5d4a454e8bdcb3d5ceac..b0b02440d9177e6c21bdd9bbdab4245ad5e85895 100644 GIT binary patch literal 15086 zcmd5@dyHIF9llGE#%Jr>`05JF&dg558hkdb@+Lx9vtg47&~`j&VPaXUt)nBAVfc0Uc+x6 zJauT-st&-&tPX8RtgrcV`mRA*Mpf zIbXwHiQ3CL1Tj)A)|hK*_}zyhiMy~+w3?R|}1pm!vfcdA=52f3bN7x?$ahHAMN9?x)_g|K@cgU_p+0Jh41Ny^( zEjV+-UU(2^ryFx6=Zna{1>tW9%HRJK%GY;G^YfemY|>V?55jMJ;WBLWFPGdaK&R&N z493!PmeF+}1HQNIUfbf%qA+T&je-Eb)CX(*(0;#E6c(1ryE*@ig|QMx&vN?pWgX@^ zecy$>^&_f1`!_kSN`${;pQCtAA9V{!c-ln+9J~`{`yWWEL$~K(8*NI;*ABXZWq-rR zZy!u5pZz53EZ{S;lJd2K4r}BE3JcEcYm(^D?HbKc^$CO-sS4PJfGfDa!}sIZZ|{H&_f0uoo|L$`R{e+mw@85b^u-?l^ZrpoXj2A)VSLcD81C&)LO}|&qkNQ?TapNUQhALCJ6}867J40`cYP3$o>8n@H zepGGBu{AL2zE!`MUOO`Va@ZH#6MWrRujLKP)+|e(aL@PjTh6Dm@HfMA#^}S(g>L^nwK;ue zGqSSq=0}`C)*ay}ZS5>Fn&E-oE!2;BxV;%US$VM*+(Z4G2U%sL<>Nbg+?ie`*BI|@ z3fH={a;mLaI3-2&cb_8K(oUAP6^34U>*}b9>KicOy=x@nBAJzf6CS^z2JJS{>8oV;i8I$MttX zeM8${s_a6PC$h^kVdI^V>HNgU;2gsG^E-K$&ATS71Man%6O-rh}Hhiy(f z5c6U~N-@jGJvYsvl|t;D62gV=V)ud&+bTkghXV86MIrX>72*(FI&_y*;CcYrk^nOJ zHiVc&`0Knsz<17bSdS;u%tP?- zJ<$x`i}HQFTubKAw&w#*d_2$$Ik^gj9 zZkP`ImxAt*aDK7m^dY}`+Nk}qaG9R=?(F#gQP@*z`JzYp3~PCfzr6j}{~xRS;1l|a zysLjA&3%~szoq@OWq^AN>)%*oof7YVhMdDiYkHtgX>1nv81C)x#HIe9JNn)+ANek4 zSwF@p#`qZlpCiZIOul4)IF2VM_v<{ZeE&Qja!m@#V?KD-fVJdaos8{IIeK2yH?aRk ztv~oam+Mg!eYni;cb*sBhcE`e$2+Jnjh?O!8pa|{l*iFiLD+2=735XN5QHBph!+r^ JMxF{x{~!58Y?1%~ literal 7406 zcmeHLc~F~W8vi96As06x2?3H2a*_}TNeCfOA@G$F!kLsx9CzGt)me|#_10Nkb-Es_ zwYK)MR$C9M^suM3V-Kqot4=*yuj=G@GM%t3#;za zV`#1kZ|qutd6xp)pA2K;g92=Lpa4U6`|#mAOK|_?z{!(8;q~X{;_J^3;gf^UW9fB8 zc;NbMeE;n+thwKS^^2zC=;1vWy4QstzW)Yq?RgOUhZkYx-%Xf%5%BFdNAb{2#aMEk z6fbVR4vVi&!^3}7W7nF?aN@`Bu;?n_$vI}+cQFskZ!M$vba?-@Rrv7jMfmBbAF=dC z8D4tk794nG5dS*70YCqI63hOgpl95VCD#gZ_?@Q-yBu!}--!?29K^OKIPh{cTvkq!_!N`aRxx`4P;!1XwlCgm1q581w%GJUbY{)5`-`@MmD> zYBQc$eh!x2kcY=^H)7pl4L<#NFSf5dkK(JuirdOCco(}fXXs1@ej6EJqU3ar&617Q zy2Mp|W-xJSoUc>Vj?c#?&b~awDMm(xnz8}M7;Iv!Emvl0CfJFYe3m(!#J9lEV($}7 zz@jr*gSw|DJ&9*$UrTvQYlAFaLzU+3F&k6k7e_g5d3kw@(XEP=)%(q6pHG-19EO3i zg%ryD_2UXohtBLX78xrNoQ?vVe!`PRRc&-D^rjG@LoRR9a2SRSsJz^6_nGu$p{}%q z&|ZtnrIwF-jGJ%)*#lXf1)@^Ms^$BNLLqfot1Edp3}dB$ICxLNKw)~wSmg4$MEX== zlFb_RlNp8g*lcDp?(%xQB`VqUq}gFL>Xmj1rl*I{LOxkH7!3ZTTONH3tKS}ldmBpo zEMAMn(r1mzk`9?sxR1sBS#6aCOse)!!kB#82LWXfvwN~`YGy+DD5$O^nk4HYsXEPIAC}325 zmVp7a8+C-&>5?kQ8iZ`{MJ|^~#x(E;`228KR2YIq9fbV4kAg@#4#sLYx_VJmr8C| zE$UhA0w<$Um?#z&UR+#U9jg$mFtc#VO~~stSgNW9_!$)rxy&ih@;G>Pb@j|~g*rtZ zog4Pput$>M+@pqdBCU);lY}*1J#*&Fw&s5B88n5;@&I=bC^rOW@!@bp7wN8Nt}*9I5>;WCCcedVQO1j*#8o;Kbjm0hZ0Yj6e$?} zIGhEu)axk|7e=W}t8Hhggu;nLG8)MBSy5^v%4ey@lduiMiU$a;PEGPkI~B{WD1_mN zP-}Hsty8|@$CXw7a6~<81`&mHIwTJ3bh=8ZZa54xUHD7FVHkZki$Cpfm^wG%&sM|-IThC+luDYWcuytj2D_z_w8X_J zc$rCh_Vg*_r-&Llx_?193=`y~o%Z=;ojFaXdDg}v>;FaJ(A#{naDE$A*#B=7Y&WE) z-!%G-_o=DGj8U{tedy*AEWRckCw@4N-5amKCm)Pp(bYU`SlUcI=VEMJ>cyN3fh`Y3 zsFyWR@2JG{YcI#}hCgt<=*l~4sUIuC6L&kYX1)g72G7L{>n_H^D?$CE8gD#5ANxn< zV(uS#*!I}@T#x$RYs>N3$FF1erfC>ncNN}zaUu1bM(Sg8@cOQssQ2r~;di&y{O*)GwQ`>RvYcvUwW1m}dE)Ko5k?UXAFG;~i@9n_4MOr*@j}8C% z@+0!QA1iLv;iauNlU^$xzoP;RE|+q>>+VhG;NM5y#Num3SUbNFTbB1xpDx0KH_Gw- z@nc+XI8WjXMDLSb|Ktu?u+GJ`DKl< zd{eu!DmcE7rLT1R<6QP-3x$j!>d|77@H*AwXl@b~3We=zPAAyy=q!`ydM&@pty-_IAaY4+O}8lIqbyd067;g+Usg7Ox*rlh`ejIUHrZ5~iuol^_o2+ew_p z^O$G!LY$f?4FoE3t4Yj+0wo^tj1pIZd%~;TX`P)OpE1?am~S)`R3wT&W*mEV0Ep2wKR+|}faN`kK zVY}95D~?}`qZokCaaT&*%*r$lQLF1G0Wd0!|?j6>Dzx1_U^nCtLN$I`|if^uRr0w^{elz zryAiyy6;Y*T4NK{1Lsj~@hR0AH&Y$dN%zywV*B7l+J$5mfmld82$?Ls~|?&zYae!AK$1TWQso zNK-H>7pY|)El*jWmomyLBpw~FJX0l6irEE=Q0CD))NV~qwplC^Ya6_t5~tOm@@1!| zXDPJ$d|`De-|FJIY_=kMskn+r>=v3T*hy71)LMRyD(Z|I>T)!>vAjTD6U&N3wT1C~ MZZvk$|GEMH1+B9QZ~y=R diff --git a/src/main/webapp/resources/images/fav/mstile-150x150.png b/src/main/webapp/resources/images/fav/mstile-150x150.png index e70f8b8b029b87cf50539bd8144f9dc59930e5be..e49299e39436fe897e9d44768ccfabbda70f9c93 100644 GIT binary patch literal 3372 zcmb7GXHe7IxBW$sNL6}E1f>6^B$1K;B2uJXs*S6m1PDc10#PYJdXWft>0oFU(7$vp zUPM~xH4%ao6-fwHKzfl2_;TlcdLQ10duGn;S!=RLxlW@A|o3r8CO2v-7t=y(7)WK+>g0B|1$ z0N>pJKrasf#Bt;%2LpEEl)JULDex;PeW;C%zywPhGcb#TUr?Xd)vD+L0Pv1jnqIvT z`gv_S!asW+3f@@%qHnK}LrMs)<#>-QF%wa;&Hw97$oCA?;%~S>{M|8$1)a=e`13FB zW5YJXLe5_oGDW%0 z#ckBTao-jV>}F)v=fjG5WiR~u<$kf30J_En&3eIe(LO=b?ZUN~C1YvBYfzo%0$^x? z21Y04Dy~MtdXpcWFr~sjmWxabujtkD$A{bG-nb^R9R@FEEkXw5PO=B{yqlU4?#S%3 z+l;3%c&ke?3{N^XN02g4Bpq8Oaak%I{O7Gr?p;LM1m4an20tL+;?=Y|)iEmV_$|5| zrv8-3$N#$55Rwqbo%cAz;W0cWNP9NPwsKj@A`z!3%_06|nwXc16WX?mdK-N`6eZ(f z!kDcmi5FDGd2(KSKlQNY?AFy=h#j?l11ifSW-(tE`H65@Ay{pJ-pX-oSS@iRmwmc9 zMxPjQIeY;kpTkJ?--pcIq1xPQQzX*rU8SpG&fk`&uCo1m<*XpUd6q@N9=LTO8_ol@vNuZhy> zq5A98(u@yilosyYvR=OD#M`oYl`F^jp;TWRZH{t5%_bdh29bUf(f^Ktr00yp@A}Sf z+7TZehJac0{bD_^WJd(LZ!aAK3fZ2A_NRu=*#REL6L#~YmvkZ z+_}@&K|nAi)NlbhFQy3%L$#_%y_(*Jo1&vOFv;v9#^f3yzbXeMb~=fkK3p{%QiG`U zuL{GdOaE;6e?VR4BvK6dPjQeNX2OXjYf^tsCPb|<$@p!S+xzB^Lre6(1A_=!TMRT4_#VAYj0%o4u*QW zq1PU0Sj_7xdT->kpLwTIwHQF|K1JJ4K_*?9)i+@N{eZVEut{L5ec)l*P-kJN#H-dh z!_{svVzS)`2+V@>lI&A?4kGeodRh*iDaF=_@rLD=UmdC}jw}MZRNwkm1>(-}FbtXv zEcm>x7yBQTt@d0U+BxzNHP?s#(3DcDN)J#k(ntVcy`MOK1G;pOWY*+@bwTXQ zvZR#EJT7pwdMm|R@ioHvrN8hDW0EPkcLuN?^Zy~r2BgT=tkWu&oL{dB%`lLw`S(D8 zms`%VF*bLSD=D*G%ypF0M>f{nKaRr)+I0DgZL>GPn-YQ>>%3C>)x9V5zd0pHu(@A} zoc}wij<4jr zzV*<6?Bzxt?Yv*1<1GAGB6oXDVI7|6NV`kFTW3;tZu`LI5yaAsJVw=q1T@{s7tQ<- zwBRB7D6T&)@-!6rgocAmt8}SkVJ`Ft9KMb^Wj6~_K2G4OcTtOt6S&31^J!&<$w<|3 zKdXU)ah%FeqvI}Qx%04J(JePQE44f7-PnhYvccdH!`PZDVDxJia@@WmRCT<41? z5$z|+2X5qv3ThZyr@iHxrt9Kve{+d9D0hoyAXPAk{}#}cn4y$Yud`Efo?(}79OWRk z89oyXE!WzjsLFKHNAo)U<96K(_l~O#DeI<3(jf*bw=?HXn17bHk%!y$B*r)L}Edjh1A# z?zP63cG4d99A=`u3&vb9@m7&vVjwAa15gg}C^@?v zS>~s1$wan_B11Jz|2mYvU9SoIUO7OhRJ3e<#y!JanSCff5FxkEX}lhm8a60d!5D7` zeT8-#M3dVKjHqkPJ%m)n)6KfLAF{6Vu-qXx!6;D#*FN@3d3rrxIgTT)h}DIe%cp%> z<(r%8xk#s7f5O<6%hXnE)S0dalYVA?TrMuEdERJS~}7pW4d1Q+$-TC#c8?w~cx(VUp-Gu-~w=y&xV3SNi= z6Wy-m41x2Cs+hr3K9V#DZdV?#qGv9g3P99GsBNALiEP%>+a?+pHyipqf9;E?6KBfufg ztKq>bEH7-GDZ(KvYZ|T9>B%8qe?xX~a;&dws0-2s9*njqvC;s(iwhFAjJ31b7J#Lh Lt!b69TkQV;5#<%F delta 2381 zcmaKudpOe#8^_mMJ%mU}inUl28S+S)BBzE#Bp$4TP?WgM)^l*+fg zUzy*y&L#8^3iDBAQ$fd|dLGS?NVW#@Xz8DOW9VI`#dS)h-&B4du?KBueG7DTcJLtx zB#N~#HL}MHFTNbgD!eRbCNE;YZy`&&^VXkzJB<@kY4<;UySYu$6)e21Ez236*MHFv zv%N0^QZh4WAFBx%Y2C}M`4*wi@>d)P+mSa ztV}9ffONL+VY_?OKh=P#@et& zQ9+AGy==`8l>#_lYJlw?J0O^%o-KrO_Je_`FC1GhE5uyUwkOy~o0YnmUjdc8u&C^1 zYl86bBv~XKhDW)-h8{3*L=cXihjq{Kx#X3}7`W@Zbt1j-rKG0ILYPXich|KJC##~_&uEl^Wvz>U!$7hjtTDWy1gWFjy7L@- z`ooK4{UC`Blk1n+%zSGEbo$E$C_7xT*k93}S=ig;F}o1Wui-;aZq-e7pxxNn=DnYLS=S@5o zx>F-y-56f zr&UW?i@rV7&8YL6$Xy8z74?5*@VCy2h6`4ceMT%Mt2IQOjpL9U>*Cvsr0v}4;*)*j zwGDUih!o19CdQ86a^e$-4z~w79tJoE;!JMR^QDJofD<;CXjpg=@_v!C%U(C7Gkd1 zePc3rIUvIdt8*=lM$qIFs})^F%7^-Hw9Lp+N?UdVfFGk6t#;o6A5!HwRq@_24Gopa z2V_G3ozOLd6UQ%s9izzkZd51v!@9~p&~*weEvUaj2S>ki)YR|*sBepve8m;P6`3m@8!}Iv!rLByYnH%r*h|@2UBMbNA%XIZTD~JG)aa!r*ZnuO zHoR450q4cF1qPPO32Psm?355ly7b*K9#76@(- zZ&eykgmp@DSe22W59avD2k^_XIu1a11u>JrH0Cb(7Wh@rNL&Wo{3CI>xYp z+VkT}+!~JgTAPm&PPVD8Hq~#etiCg}DS`9D^EmBgMTH`5CXm?ii%`qpg3)a~zh7;4 zj-`1ZUlRxoNj1BzPSzD2d%}yRA_)qI`O6C}$@o>fPLgr@Nm4H^dKez?%BN#zunzFC z79ty8t-Cs^&7Yd_?PjV4YUkwT-xd2@b(=FWRM(P=uQ$7hjyhBvG`+wWY!%5tbs0sz z7Mnk10hsY=cuESkeLKVS3OKL6Zqn_-8e4Rz_Y0)1xHJ!!+lYd%xcV{olb3|GzL>xJ zdQ;IGsb-L|?X`DZxqZ`OF=DE1z!2j~bb>dWo{qWqa@AjN*4-q)O)n}}s(q3X2M&9~ zO9-9}jQh%5xjOEnH{+mwh$4DIevY7(`WT)z_U*m+YW3pA($o4m@y|Nqjn&n2vvMBj zO+@z}b8fDtqj#FlA_XRohLp9e5<<_x0(h;# z&HTY3%9lRtOlTR7cQ^ZXZQ)c2MLfZmMrETYei*xXf8i1j9wnyG*vIM70#wkwwjjVa@|&Eius zMGUMa;zlP}YPU_}Apu9cg{=TsK=uKUwAyUhM08EZS4KnFLJs1zq0peJ#7qgy!zMj{ zp=@sCH&7{Aei*)8q28MxF*|JIbzgFzM+uL^gf2apVg&X1Q~*)9?Z_6!ZctydYf#xy zmUJ#90K>2IH}cVtiuaIfB|!}?(*}uH=|DZCMUoCYyF&f2->Ujz643)zk1N1ns?YBz zJ+$3No==c+rH3B$Kc6q_gtjkl3_bePxvi=hnloS|6Slb4_#g06<^TSke@OQq{4e+n T_yn|ffUhRHb7#`)CH(#ii+Hhz diff --git a/src/main/webapp/resources/images/fav/mstile-310x150.png b/src/main/webapp/resources/images/fav/mstile-310x150.png index 5677d8744e8c54b9f8295653beb26c26b0b3a873..162c729da375531a53c503653f9493ce412f82be 100644 GIT binary patch delta 3302 zcmZ`+X*|?x8~#(WB`5njq9n&2DeG9nnIy7h3}wl7?2esrgt3mTI1vqlu@r@248uuD z5e6|LWXUp&eH&xUJLi4hFYkx<)APH3*LB^`eLu^6KhKb0i$snn$5{XX&a&Gwa;5p< zm)y;5ECC?;DgY!T0l+a6N>~AaKU4vL{15=N{{jHvpci;+T_(Wl1~D@M7>MjIT}+ET z%)-)?ec=qxMIBy8Ij2AXI6q}!VrUyVNtuqzbRLWQxV^m5PU$&fE5`C@*1yxN$r`8= z&rdQy{b^%fnfFxY`PGX)cF#;cNpspTuF3c(8M#beNf)SXuC$R9lYh%+YscX%*THt? zw_h~R=}bIy&m5S1BQn)K1Dk5HVi^4OJZ3X9;<4BA_icLoX^~d@;TNh!d-lfnm<=ty z{B6#AaD!{sD=e3|C2Gw2n{E2{%!=wc8_$hl9IzJR%@(xF2#|CU?_6HY#@C)>ZWx-4 zeD_4p5rKuR?V(Duk;x$WgsM4v*A4a)q}7rV1EUfOS=naw<|_ReLts?A+vy~nrg$ZR z)Dyn!a)Fm}t<5t0g-_lTB0x72?(FWzSJuKGM$JMxukTF^OW;t&kWTAEpYU?(n{e;; z@HLc-<*;)JhFbLW=yl2Qly}?;7vPc8ur>HWxYWOsB(*OZptlY6FI}xWiQ6=fjhZAw zj7h+yP~F*~kKJxvo7=IN(jI(I#tHocy<#@fenlJbNY|m~JKj4Pd|4fv|INR17_QQT zJf2}+z6D%@_aS~Waw7;M$vAHMozKB>rzR^-fJche;c;H-#E;$EaqHK+?uD#BDHXwk z0MWT(T}J`l2HH+7vU4y{P%>_Ulv2-ba25By>lzng;WYYLZ|f0Mc3Jr_beb*Uipus( zxp(Bam;Lnn*gz%nBZP2LJ?FO`INCd^(MV6u1^o)T_ustRx!?>F5XjPP2)=h!7d;_p~* zznUBv4pJ&HVaHakP#$P2g+2K6Yqu=;t^#M`#9a1p$shu~bE_;6~eQgjTmzPhH^FC8R(jdIbzg=%vss26^eTGv^<_FNX? zv9T|OsOK~|7g;W)oStIXP>V;5>AQS=?-XG>btd6{Ue`Uo3o4P;c4n^YlBQ<7v?by` z@QLH_=uoy_@AbYYT0a~~dL}``VgA{u$48pHh3jyKJ*iUt+J&V^e~aexN9tvMEIHDp z>+BGt@*h|^SH^6=yO+{|m=y<%JK@f@=?sWFeZa-;bTAh~%}9%@yh%Ip)#o=*K3e*4 zA^nNVGE{HKU$%=h7fO$aA2D<9nqC9*!p<~idpU)EQaUc*u#0#_#f7g%5voaJ4dxiX zQh0%H+tRhpZU)PA86#Z+&jpAshQn(-j9_msN~yFxg89A#_ov7JeIa#$JDLt_rmle? zQI>npRkkMcBt;a&x5QLHGz{w}mI#_XI3M@0*mi57Gp%?CsQsZHTgT|7>Dfp-UpPjW z^`r)IeAlORJpY*Nxs%XJiP#LG31 z#I(;$^7&~&cmHeFR-=~PE$@H0EmO_(9HQw{y%<-KVZ_V%4_9jna!^oE!L2L5ha&27 z2tRF7QOgZr;a^paPU7xMMS9n5FUqGAUP#w)_VarJ%fg-lK#K!X64<`6g?3KN0q9TW|ZKK5#+0Dx^5oQ`fN!-6i z$uj4vY4;;@wi)A4c{5Mb!kYdG`p+Okw&90;CV0~_v*aZM9IEli1b+dx9^P7P=@Z0v zl7^B|Aw^rN_)Maew98p2V;3l2!zpR-6yJ3TjK&hQTrtA8tllm*xL>O%bbkNMfluAb z;b{NNW8-XL1O6=y_(wlN!~E3-sYp`Sw(rnFKVJfuZqLizKs}9GNBNmo@pYR!Cqy)R zjzz$s9AW0Y-MN0+?UTH~-9@4u@JVE72TU`_Vix2_;FX_c3d}?9GS_Psl?DIl(Lxhy zMKCYKIb08X)nScKaZ_utT~{s4jw7v(ma}O8dL%*vJzr5i2dw%@$H&cl>8n0d+f=LL7~DD&DV}8M77?s4<={sxfscd*)Og;&qLU zUUynbRWS&l-j8qkQdk@hhA-ObPkBo5Z`(3z7y|@PB$yoYT@60hJ77cAP{j@_liq2QaE~+OXfZX}JhAY63^GAtTcZKOnP{dP zcs}m#R%@)rPsljJ*nu1M=!E)IMQtcL6WW5GI;@NxJFd|&B5|XnbqB1j$~?5vQ|`p~ zcHP}OL4t+wZUE05ly691YFF|Pa!lpnmHD6XEs1O)f0vwk_Kg+PxNknllb~*Va6nJH zNZ@^eg~5KlCb%yz?O6K&UT{;kd+mT|ht74RgB$9{pT8|I+xMp9vuB86TH}!=a!JbK zf8#OgkE|T>7h9d0KQ>#&o`(+h1mW4^m5?RK$#ZlI0a2NWUQlzuH>rel$r;hz?-;sm7 z-Lfy|#Sb0dDdS|R+h{%!Y$@NHExc7#modBp;##C*e}GV#4)rrN1?8- zU94A`e%5=ZR%^o;H%r+35q%d7ck*R_oSI={bn&1|#rGsy)Su8Yx-D0CQ(P>s6<(aJy zy&#r~y)){w&k$+7Qbp$RQ4QPn7U-5mlpDEeZiuG!imW{;yr-2dknt_H7F; z!vx#!I1@w5xXl)rP&0JOO-ZVj*4F<3uRXp~pqcWYWwg;iVcacY$939)r@j5q}T4uDZwOmScpur>ENIjlqb?1n{ zX(RVrIKz-(S#s<$@4z>_uYTZsHsoDshuTG8s-xqAp4IlG$q*veeV z9kY(fuCNBPJ`iozy=^yXNhlcvH2zI-dfm>iFe4;BGDWs)xZuYuLlvS$mCwobTf(-) zp)fBEQ1NRPZ&1J?C7!gezAhEzyHX2yBusVmwUY&u2&cc%L5XLHdnb>;)^l8aD*`7^ z8?Gy`vB0k_ee3L{ef&i!@JRVEGfo1ys}->S^EAqmJtDyzrH z`b-ql{Z*DZWnvSc!`I$ZiW;(4^YI2v8Z}>h``IMF-qpjHH5<}o5FqtNW~VaUASx=2 zCp%R~6`=$Es5S1@sE#L;`A0tx$J)gWo38}3GT-3td0aOFJbD>t2G?#s)3D!BWd0gt z!%Q8+JRXI4YP*McG6&$gs+yLPs=AVzx~+=;8A? Q>i-KCyuz&WA94l$3oM$582|tP delta 2500 zcmXw4c{JN;7pBv8T3gF>WE3?;MiH^qRqEsv2rZG%Za&;cySW6K-klMbzR7)Nf6U!6YQM-T^4Kb3*L-`+7%1m8A~fd3uTL{}5Y;63g(t2)Oxl@oU9I}Q;@$jflCm&734>a z_2J#NW5-xSjGtj?NU&)=To4Z;^(xh6FWl%z>J`jcF9Q+bArhlW1`a{G#0ZmPn6kqV znmC-iX(}AHHfZ+8O1tM12OX#Ab2K{tjPxEd?5S~LPXv~LO3lgm5=3&|LMXB#MnyTeGquc};ed6S{@XApkdIaBZ~u7t56{Tae9k+quo zx11miLO>9_{F35I@t@&2LIV7zSi_7^T8&Ro1SB;RL5xaLe;WFXimdv`!1{mBPzGYc z!oY0tN@Q3~*I&S?V1WuL0O{jSuR}<6TRPn4k{=>HIW?c*9x!UwlyS(uoPj*ANd3*X zSdCX|vR$h=&yiqaY@49uf>9UlXeG`BHN;tmQ+Niy>E(1$D+Rl}e(5jVt53L3LQA;1 z6rAtaC(Q%yyQS+Rel{y3BTunARVQV;t&OO{FSTaq!XV6haj)4I<BI zWhG+?k#;FxtzI0!bQ1M-xw#siH$H-KGxKIW~U$(uY=tX+zWXP*4kX{un>6W?q#7n>vWusmnU zBQ4bNEEaX&dsv4h3#m-A-v09xQOe-}v)n4{O1Tj(X#FF(EqXK9o4M+gQ&k{reUYsv zkrjbX-n!+_hRW`4OKn1rA1=nTc8X@wkq1FOvitAsJJt1boaTR; zw9p990o6e(fjMKvhr#g4O+OZ~>RXvc`hkM*CFW|S&%t|)^e&PaJo3xRE&K^ za=>fzq7nd!d0v*XS^h2eKfD!~Wc*o=%VoNopDieccW9m5HL2C(BPV3bTHkv5POX%! zO!dCWdsS?3J1C^mGBF-@zlvAaZyz1ii5#k0LhCpXx7|(^E57a4cmW>lS^Cb_<|Z^T z7&x8CIp!&q^;UJ%aP2QT+`K`MOEThCvjeheAV22J@KV3{+xqa?(1YO&6Ky>;+RZ!K zt(0n@zRpqEmNc90+YXkdqC2qA zyVUTU7s#K@iTQFhNVwk-RZF*qrG03^zoQ`P~nMu(Q)(kqwxGY>1#mDY&z0I zg!aRY!?0}Rov->A9+;)geNMkXr{E$w)}QZ}*t_LO*^aqI*KNN^-P^anj4#FZf>Oc>43`Dmo;xqH)kY-_33iw^f`#*#+RbXRO$0xEfKiM-0_fabg!WT4 z>ar6pIj+S&rqosX>dKsuxSWp@5z3&wHk9H&KmIKKy3<9z5UtzBFQ)9l;h&-kH#AgN zD_&lDWY>za`l~`Yf!!4cWc1!vEE2F5Vtuh{4g+yE3yFW#Vnl=zWYM`U7-yA*sYTU# zIbT==7jO(}EKn(epSVJC==D?c%hlUV%|yyY{6VT9UhYF_d+#FQtr{{d!H7mk#zd>r z`|XwGQ+1U`jZ2>nlylTxc_Z=#zy}&V!X+nc(=b9>R6s|nhYm1pdi0Xd~ z0nG!#pX?9^w(3OsuCQ5>_k{Ah=E87@}Z6`nF~Pf6Q0wvpNS&<7KpPM}n|LGzVIA_<^*j)lLD ztg*QmGJ;bWJ{ys8zi2Rzb5+%FLHDN#OJKT^x4hC3S~ULmcy*<;?;z7Re?M@!l1D*M zj|ZQi5)|oUL3j&vVb*;EO`*m)O@KGxZEf=__hxYy9^rWrw8+VWrY+Po_=8SPbq;sMT9 z+!J_V+7RT^To9cbOsehKGR-#tl!o(d{_6= zp5q0O@`9VYrkmpb4b{yNt-nhSpYAC+0Q?ZsfYPvOy83K+=d76OHMP2D;;tk-2h8lH z<-qc6A-~mTpk$&59+&F%J@7+4s?@y!+4_5D*4pKZ&;=eKS2lBcX)Un4$#LlGSD@he zTZFtUCCuO?T=`R%Xa7w+x1s&3(IpDW1y<0lf>qft**T!&#$VHSiJ=wrsS^GVQaKj5 diff --git a/src/main/webapp/resources/images/fav/mstile-310x310.png b/src/main/webapp/resources/images/fav/mstile-310x310.png index 387fae1fcfaf648d9e060ffa778a9828591d2a06..ba4aa631645144cd3148f73b838a8325a24041b4 100644 GIT binary patch delta 6621 zcmZ{JXIN7~w{-w1iin7SbP=QzPzXo|rASArln|9Bi4 zr1##N(gL9qS|Iu2``+)z{c(StGqYyxnZ4&cd(S#EB1w>oXQ#UW0)g&6&@)TmV+L;8 zKQu7_fdT|UppXa<=#&hFtbss2QXtU!6A(z@GYG`ykzW5ui40Ia)6>=fk>17iG?H7i z-Z}=Fw124>m=qbRMHC-^KpfdR8tP_#Q=7B?=}^av)!pqviKHU4MBNVf{re#}dcKel z2Mr2srh#fs&gqxf1Rk3lLnLf?--Mn0DhlOnd)2O5!IiP4z*{crV1P2_9)H*SYhxE4 zQVQ@D3OK-`1WXY0C^zm4^vgN>_jw6?%werdq+kVZ z+FnD0K7}J>EQTQll`@vnBVU~$h;OGp4>Q&WcuI!#rhb%6C40A)vb0by-tF=KWH@*& zBiAvt)FQ@gGcv8vh!&+Sx%N^y2Q`&^&|b=ArCod3#%%yw|2PF6*;Wib158t7H%M3& z;(H`u{N)~KbU1+vQ5j(JCyo7~k`oYP68#u<<&Xk2L+Kit(YTW?<-;Cb zkk)wdogc`<^^51$Mr)dz=D_O4{0&cx79zE8z`tHCznHH)!u<58a~MgWPcHOu@HQIhEn!yj#BMPDP3uqbQ=N)g`JA4lM5EG#c1@GR1wVY26I0NmI_ z5JmQLrFJ9Q%V)AAXCw9{p7o#{@^{eWKNtgEGGqwpTE0VUA5jN@3Dw<-9D}i=$1u*F z`@eV^<$U8n!j}M1{D)_8?!s-$QOvLfnEy>jcO!o%oC<}MgWZ~sNO({#sRQ+3ei%VW z+v2U4bc`dF_z|eCzR^Ao6rl=2Ns8{Nv=eMZR&^E6t0$DHo;-a0W_th1KHF>ci-%Kx zY-ZD5HLBu2PXqx}PX=cG5caO@$~IPN1iIl#J%QVK!+;muyr*5$(PSHE9uB$kB>)(iha3e6*Iq=E%fPUSlm+3@&ZrX zhgogVrIp_`E9?{KuHBz)1`X2g49@Law?WkV-l|#7PXM4q>Elk2#hg4^?)QJXY4>^{ zR@?H9MYnb>_aeG;dDzfd#`4Ez8iKVJ+g+7?UC_?xb)1PN@f&f ztPdC~;Iaug82(oF1ErKZ6Kw+?8L1cxyQEd?5W@M1z_%T6Uj1`afcC(+FUAy7^K&%Y zlL>4!@YPDbMmZPiSmylZ%F4u0!Em=7gXs0eL~7DnlPJ?a zTS*A^PVOnTysr%MH6Q5Dj5*&blPzg2&s4~*mW&QfmCkY5aQt>;5?}&P4y1c^tK+o` z3ryd6y(na(msGp9n8;A4+8Xc>yznY2UwZY69`|AxKV6D=2c~&uQNXbFU|Go?D0-wb z-txfrD%q(FFFn(z^P%8WrY=;leq+E_vxuk%wGQgeeG<9Xg>1D}Y(Zb;j8Fxl5qhj# zU3Gc4hcqAR^?6`TEk8vWzx-#I8_W!m)zuvK>idc9JK?%${??;2%(V+o91v)=dVKeY zQv+Qq-poYizj&k_D`CGU4%{-o>(;{zw%JQ}&M6|VqTDpl>jitEBhz|k4ZQn1ox4l! zEpo)mh5Az^RyF+S{X@SmP5e=E=g#T+x8@v_e2qk;$I~jM)$T}I^Y7bNv-g<5?C}SP zH#D9v(o65YM(jS${1NTt_F`CXxr8Pu+N1MrK1#JcnFG4@JuDbN6!Z=MOrN%Z8(AUL zsrhXyo<7J7WMAD#-(&fsb^fw1`s=-Dx3Bk^z+d8V!|tALFGjUzjho^adi?{m`&`l8 z=eH?A4_5SEORdVKzVi3~QgmZ+d-cTP0OM3geG@UW4qIg(dlfh`aKvQX8h8fp<2XIz zeRgy$M15ygq!^^q$OYjwNo97%`IlbFYqjk%4qgH&0%;_e=i_JZ~LRhU|T@%aEIDa89ndP|@W> zbep3H9?0^X9XRp|Q||8!RGggKqm zOWw9C729e}U^`j-^xSH55=o$Ly6T?@B;6fhJavkb7@C8h$*Lw*H1YsSzGWa`O4tfU zq~*MP8{8J|B1f=p{3CPZr%z5)#urNWW3~*q_lnOS)m(lm8JTiSyQMqZ&mkE*%MXhB zT|(4Z&#nO!&zU88XU^FgJF|A%LV|EdNytDdH75U1OX|x(fi>GZZ{GYk`ood+hqRPL zv8v@;*4Siv>&fTcR&{6AYE1vpNxyP?j7a0-jf=r4jmvDNncCc=+l`WAG4_|WU#ieV z-P++@n@;7LsD+h1PqJ9Mh;nJ18Wj@q{i*?hTNDcdE(vo#98*hH7AUvIAf2TPV+9Hk zdQS3#@9;5UDp5{8h(=#{{*{6U(4t{s)bpQ$h@V=8<>Oa)N7_I3A_*Who!T+p!|JB! z+-Dp+m`oPJ8pWr0X4oR$F*aF+G+=7aD<6`$Z)3VTz&gwawEVT_l@6)cc|i&f?7Al5 z+Mobz7BYuC{hfR6vMv6y{NUuNq+A>*Wc`nIin;mA2 z%5McTTWOGCjCI9;L9X6V$%-EGei0_vU z)@JGPJvc(c81~FhL=Vk=GfD6zR`^XWaL)f36#!WnE{##( zCr6T!OakcAAJ+WFBgHhA`E5OZI*%qyIjLm%YATSiK&xrfY5n1z3CFMGoPKXEHk$=| zTqR~gem5NA-fs%^BY?ogPy-B z=AYVyTzE?*NBB6$rDz(v|8u^**%VM~^G|kvWr>sV4-X=SwYP3@lFxn1!Q#HLO9LB$ zy{ns{;+`3*3o%G(fig&N$`^?%4$<1D!;a~~a*u6ZCn4;QSVEq-9kR``E6RuA4uw8r z6w^IK6@K_vxeY-$9z|rWKgvIZ*C>qUfqg_DRdPT~dQ%H5AcZ%O9>zefJ0O^W3}k3k zuf1%~lh-&V^segWM937r!U7&MRb_?tHh$-G+)hk(rI}q~f)Qp|SgeqGnt@C&sR_kvxB+>{fV>pZvlrEt88XQ`1YOxg*wlKz!giJ#77%Qn|`Y zP5z)fUN2av`7HJhl5nkQyS5J|?r~##mqDd)$;SW>;ESn8eI~LN+rY+m&a|mj@~p7k zUDBI#48#zactJI^LB`l2mF+9=U(Jvdd*aruKEJjiY#xi3yY%?ESIRsO8Wt+LszY?#iJ#rx z&^_`NMh!X5KW@S;5|m|q=PAv5v#tc|dUufSY}GK%u1VGkl~3^ixBekL(dV8EPc{#G zcl)e^Ew_c|S-IXKUI*X;LPeK3011!m!xM|RC}ed-i6%zrB>Tz{GCNAhhKG zrmOlCT)U1Xnm-vu1i^9eP|>B|uq!;@^qyG;+Aep8H|BZ=;jGEMlXh$}YDboHCN#wT zCYIo=qn-&ArFpv?4RM1sTHF8L@NwWrcg7;jJivms$nNqh%3|QiGz!$hSsOhj-EX{Z z29Pcpy@!e!o1YQ0PfEFo#yDLso?rQ4c64hyjz}@;qW2r;;n~2^cF_mg0U@QgyTQV1 zHyT>5T~?d)E1jl%yX+>F9MT%FzGJW1T?bq>?l4D?-xOVFYuszM!f@HgU!lWCQdZyp_4{Y=;C!tz~9f|=1h zHye#@;AvVIdEYPpkk2+gPHL6!61kCL3>7+zv{edvsejlGmYy(F zPMi9!zW*#GQi^6&k&o!}1ew3tpcGLqm8H-+cMcdMI$$bg`z>!2>;0qz!kwFpX++ZjBmR_kDS zsy_;Ljf6Vu}i| z*|{H*3z>RUFp3UJVthpx1*v(wNQLZKF=IiUqD|$iB9_HOb1DjYYq<;hmBljrlyh9` zAE1E1eX@*!T$TFGVYW-})%fduNH->SLi5&z#l%OERDHJV(l@{c6vdRx;4lC2=!>2y z%{*^r{HY1mVj@{d`R2Xrewe|9?g;DEWk?DAfWbmhwBAvd=u8$s0 z`1~^oO{4nW(c3eT7;li8F!epd1L&k7N3WXJmZxW6jqHSje$$&7p3M(XLGrpLM@ylF zu72y!$z?;E3 zivFl-ver|tb(jo5Lh)=oYz^2j{@i`E!6o%UXfB!97FN$zR{2jFQm(B&wY&zKNaHxJg}I+{9> z^dGDk2+zEMF<@&0%Z%nJ2xX7De>u(Kj?nvloAy9Wt&%lpgasp*VS^tGNvl(?SJi<% zWmCK37g-}e8{Ek6o97?*$7qJdL}F}9b_?eQM@XzjM`?5imGo(`)E+%E9%kNRil5#bTc>bp}TwZM$ypC?V`-hd`h4= zxNFi+_0kQu^y8H#0+@;jD}7(a?q9Uc@F6*v-RIo=%>wHEZ$c0Tn#|{UfW|+qlJ^bx1 zc~q9+$HL-Q}GD1;eF$Z2|xX4-Rud|=4Fz`Xa6L}y*OJW)@S@r41D_ug)tCoQ!9 z)V_S|0u*=~#VL(t?MzW_RdE$!(_ZqXQ29|A?KEDOiFZY}ygE@DXTu~jHFO-Y?JS%W zWTlXtq(jUyH(00{ode&I`rJk4PQub&BY`zcc)^lY7CV}eB0atzPal3`(|L2@6j0wGM zd|BU^qdI+!zh2Xxme7c($sBRVM~uOpJA}+;3kp-qyu35}MrzelFqS}}BC|I3?sb;M zyC3QdvxZh7RdU(rD)asgYS$qxS5tI=_v3e?LWPD5aN}HS0Ae5wF{PJ*nL?@*WeIrs zbLbE+de!)HM0fcLQ@xg)i#Jm+TuoUa8?$aw6EKpe!4J6~10V83732udCrsq@uN$fa z(l~@|1jQUN>}_fLV{k`%JWTO?rTEb~Zbf+$<<*+jEfEm4XNf9~8#&A$N19jbRoB1{?TlhrM*R~m$JQUW6Oz_16@zzJXcZW*d%s`Se_Wo zUPH5iK1Jjozol$oF%~}$9+V?|n$V8(TyU};^#n>-*TVy{3`M#|&b`mAixu(quKG03 zcon-sJqqEagD&+ZmM*31y7jqk%K3^uan*YbUih1~MVZV7!z=-1$#C0?M3n2*q^=0|%+CKsR{0%E6C+Ho5y%0H^V$sO+IoRP&`OD;-{ZT&^>piB&h z%M#{u@a8y;a$Y;WYD@@qmG5E1Y_A{1*GH-)k+dmf%S zC_Po+3l1wH|IbUeEp|e|69rxB4F@|pO0*#GZyck-#Sb#pjo&F$S#F^0Y*5g$qBLjq z|6VxQ0wIZ~BzZY-?Y9pQ^3@3FwzuYEZ-=Mejtcgkj^qdEuGAfQ2`OobJJM!%Qt56@2I_LpxLWGPJwn1OzH-3%-YczDX<1UH;_L;8*Y8xupGl5dh6NC7UI+oH`q z{CI9XMt65qW51)r3*@H8V1>4eFcD=()|T64`B%Xh9?yg!Y?X@7M|-$6u6@lWl|j>bjgcr+)Te z$@}FU=U`i#dm)nlkk8hEkaIf;2JruTf_HGlfR3irOGHCB!7ul#0JLOZShcOpWUC>q z*b&HBz}9zaCj=@ZRrY&FBH5$8=nN!23K9&h1Hs}UW7E0w1dq`-d%Yj<>YpdSB7qub zUI~I)w>#JV+fqid^OZ-e90Fgvz`Z{NL!N5GXfnZ}IwpR^giZGmPTfad*e~@XfX_aU=p^RuPWkGbP zh>8Cc4uHtC$#Pc-+51}{0Z{M4f7LMD`J2e8F`ZK|fuMwh0$Q4X+v!WJCRiUM=|gnXN&gwSQYhTJ zU93K|D_{q8XznukXcZ@0U*l>Hl+q=DzL7`Djp^78?D>`&iS&h9J!ebS5ZH+ivhFK@ zno~_k2n}+`g|Jq@8Zyl@=kt0$AO8j;ms8U^WZZvvyBG}owA^EjLWl$jjM0sc5e&$; zr{#YCyQFYX0f=@)9R@cR!562@;Gq&rza{Ddq3HA`{io!Hx1(dQy3qmN+*xvN!l(0q zWokA_8#}aCt_}+-z)0-9FODkHgLFflZT)Z|nDTf~0Ql4MVOcxL)L`MLHQW##|A6sYHPQ5nhsF-maM>FX9F1D z;Z<2B#GD1x#)gafbty@~IH>Htps=4Wsn^)gTj4|p3)ht^@SMOunfKNZ+$aKHv>|Lq z6%_4jQ>6gi?p#MWg)2xpz3cfb6$~JwvUX`Dh$2K4|Jf1(Kl;XehI=> za%t~L>hmyrC8*0(NE*ElIX*|CGg6oeQ4-J*J`eXeAU?GR^65u|K~L>!d?f*02AZ9H zaarenG6HNhgOG~ci)?iSYE1ekbCOy^5NO#)uhW}p=RTE6XN3f%vbIfDz-O@S*Hx8 zniQ`dSIjXn9t@xyOtDx#wc%o$yPnFI!!BR16%VN8Bueft7afwNWX`cl$4H(XTu(Jb z&<%W&MphnS6@LLv4l;|Lhp|JSjxlqzaevJhS*1xROW9NEGBDlBcppy}>cD8O6ROak}|h zo+l!RkdE{=^seROESy|wf}sMP_x&LKgtJ!MM`^de!Tn;?i=P!A3IIs5^~@R&o%bGX zNscKb!B8zVDlH*L9GFdPOrMwZ9DZNQ$3Pb^>VVy>>D!TiMr>!sj>DK}b9hLtq#Lv8 zCCPsv#V&yA{@{?H*|6Dis1)LgziZNQp;A$lv?0hQSdTV&nL8EAv#RE;aa8s5rLIw_ z(*kn!@LeyFRE!nYYai+I#s>$CCvtS@H9|fXR++3au&9sIw!|>2`%5=gMY#a zUpRG)KAcVa`yxDr$zoDC3Q6NY1!`fwlfqcPzF*Y&Ilo?IR@NlXYrsq-UmcNyzm}OB9?0e|eP^s%3p{R1{T+de(w7!nHqqjZ>hM{*2^$;}uv8D$0d-%?KvJkkPZRds%aUl#Sqcpa=H!JpZMFW+6Ej^&a(Cgv3iXQ)+z z^7~GL_A3b^9>5=uXYSHxx^KPLnA>e~ zBSy}<=Vvoni{{8YcHme)D(zmGz+%2xu*ysmIBnJCSDdp~#KeUv`LKz5qi??o6LTX# zLFM?b!JO55siH2Gy8)^dc}gT2Q_IE*tf=cQXFFIvHuxPiPrh?=@hYhGN!F>P`HGhT z`8NZvOStwAYV9k0aMC^K)XU21#LRRxJACKvXyX)pJI4-|BT+1CJ?sp2>8= zn$c;Ws1!`*?7!=ZQ*LO7h)^B)aHM|NSr|%ZbCjKlGO%+1#3B^peZ1ZY1ZcTOYq$T5 zK$dpkwFm?4O{8x4l1i_O(?C^5aeDq<(+aBS1B#os0h!G9PYnW?c(v&$w;e4mS`!zp z5~f^NcT%ej=>p7!PAnoQ2Du(UU0)*9-l!gYj(J-E*Z8(KTJ5t|=%^S!_BHM8?i><) zaGW$-KlnN`AQz|E^6GDd)n1&D^MD>-EN>|tir|_gsyn8?Kd9)V?;S4dq_d>HbR2K# zkKEyi&aXA6(oq>GgHwTI>_=I*FtKiJr#<#ycr{kz9xsZ4?~Sn5TUP68x@h7&^^fx8 zXH=!3n!pd=0d1VceO^=-RDr>$nWJnMmdP8~ax}(oh-7>z6z-f)5@9f+KlLd+=G6Wvhon zPGe&?hc6fi2fqov+IH03F5|J4+7lu3T3u-SsDS$(#~vW6XJFNWNhN07QYo=LDKAT! z^N*YX5V+AMXzqj%I9cnNG4NOMu+`?oT$zR^3)3!o0Qx@@Sf2{*xAN6kkjI@5>aBIIJ41$CSy1E7g zf@!e=F14+`nwAR+&*IXNpkmBBHvI}BC?EOxRnDSoM)_s36?IoBjjSr#^KEoX?T3`0 za?N#7{-Hw5R-JE8*E{gCz0uQO{re(O&Va{(^6x*nwj)Q|%LUIRjMRbqgi1ZBt&T(w zM^ybV->}(~%;Afgta6B=1^7u`}6$A3$YTx{M^v^=i&qkXmCa~n>{5v|p zn9C#!PL{712d94=y(NZAYQb8vxQWH}WJHq2RRU=KFnByI1aSpTHj{N&guK%mX?TN- zuev=Uoh)R#5Y(!x;B-L$ivL$QUL`b!>`8U+`Nna`rzy6df=*o#$G=cBIq0Ns*XbWt zrx?#?194+NOhS)=ga%zVL9jR%Q&TZZ-S}9yc;&V&f3}k%5C87v`aQq@Zk@`C&rwlr zxR6kt(&Xi9oH3c*wN!}ea{x_;;(}B5s-QC-R9lc!p#ZR8lDxD;))M;TWqbXh-zDq6;u~Do}jWuQ5u4jY+X4*F!mNnBt7+ujK>&?9;5%Q;FZ9iTbCL zm5ioan0wYtX|JIeYRs(7)bj^{tC2`SXe_O>6NB3hnkfe}R&6ZTiZ9wXXudu!MC2E# z*!Iko98i^QO1KoWmBBokK-Cr}I5dh7exk{8op$wCCrK_*~ zbm`~i()rI4_O?tOw?*PFfE$79aSx#+00IS98t=d~Ek;?NN<2?jNU=-_kjGL4M6?WY zXklREQ@`hR3bB%#$|~a*W!}*43`HCwc>IkhuX;&ao}}YD7h^u(XIi7o-A|yr9qNXM zLs5#HyN8Gr`gV!V5-2KGcO0iMH)*n=(!-zOC!>>#RM_Q@k$%@h=>2xMZ_ObB!5yU) z3BL6Y8k~>ELS_F7-pKS-K_#!*i2JjxhQbl=RhRHb;+;Z(3;2+^dnlAM!y34DY+=?T z*jWVJh(-E^4Szhw>b$a*9X(mD%e~>4wBMAya0+=gy9HibPfGD5QQiO@-7rsa6(Vt!F~?OEyWKxoIf#iuRpN ze-wxR6B_DgvPP~!Isa*WbMvoAFvXMH&QWw`YL)3$)Qa+J+|%2mSZZeYj(p^n!Mj<=Y9ErmTDWzn7~ku^(FhK`PV0S#+YUsp4V-h0E7cRzjvH#w zV(uRAs(%6wcsWeF$M8B%(qE`M-mL@OH!%~W5Q{XUWi(24`8+XRsnTd-4?wjb+g@&( zTW!_-MXW1li(EA4IGu4@n>xv<$WZEJ(FjW4?s%WPtEa|A`4GdEkNXrV?H;$oVnJbi z)!0iozXk(MlcRtuHpB#YC^eipFdjQgQZ(nfJ57>Rep|Q8xyd7XFOnIW&e})#D~*U- zO>=h0YXSEYj+^K7m79yp*53vT9JsMLRB9`HIV;1?*!0gFJ)G?Zk+z3lQejsqiD%|W{k+D;ELee5nLd@ zQdX0S8%Q7?{OQS?!xvDBG+pgRHNX9ogZJjI8()YK>LCB&~QDgr_`*b7I%Bq4zF*Wxd$v@gzyYec6ibg>r>LsLZm@PxFqhKkMi zBZaD?OOL8Q`WQuc6TZBlRKA!B-+b1*6h3SEund`JOg=wpW0}O_cKUqP_^|Qfs^|5a zgw>ruo@>JLuk3@H(>9lt>MCuooMNPT@kXZGT_{?I*}oO&^TfV#k7LeYC(SR7|{+ zATVj|Fo3RXeMiaJG`2hGiiP&W9x&R?`XXxaOp01W@Lx+i2J6fT2)hd~u5iCEGwEA@ zZE2@`a!LRR+0uA3T@*ntiK9luja4HwPlJ+I$Xe2DzjWFe^`~Xbx}Fpzzzcy`h!7WufVDG`lz zxYnJgc$V3p8mz8HKca^yPSDOqu10A(>WS6Yns&18fvpY%A7o8uoRgFXBbzo)I8@$99ts`}V86p>p?qSU&Ny32TTE`7q#VYQn^>m8xji#^D~kW3=yi`( zx(}Hzi+e^PAJcPb*^pj&C^}?qd-*Kj<1lta`N>+W=Ktt)mwk?2@+nqO|LlI`njR1{ z3Y7_+8`FUm=VAIA$?`&)P+_OiJ4^5LLKlJm`}~L8*#7^+z!xQv^xv-DV&rVUX&5G7;# diff --git a/src/main/webapp/resources/images/fav/mstile-70x70.png b/src/main/webapp/resources/images/fav/mstile-70x70.png index 7cf23b9cbd6fd25dbd947683585bc164fc6d47b9..a38d77d7dc626e56e8d53e574feeaed6182aba05 100644 GIT binary patch delta 2224 zcmZvdYdq5p8^`~cMzP54G@?}+N+yjg6Dh~YVWWgc)+UF^VbQ_eeb4$X0iIg2&qTv`X+p8NB8UOdm|dG))l>v#QL{a$>N6pFW{smnkB00=ZKvb>E@ z0(TJX+?)X5${qlq#sk3j4U75-04|~cV8#ysFiZeY4a=!zI3@6-}{rcGfw; zgx_$d5PwBp_35H5izG|DgG!k)=V}~OZ#n!;_B5vimQEGEAPZE&)(n?> zoeB`lfMY{XKR%rOfHq0xyv%ANg;ob>Js9Fl)xXH)M)enwf|?Jl9Ve88JR@I4L7q^+ z;a5K?Pi!?W|K#b65mY{KEMz3Fi7*wJJGF)2E%*89^KA00dvskA*fb=n`LS#|{c^&M zV>}rRiDBJk8Y6XA%mU1AJ~I^J^8$@T%zH7s>b!EJz(@PcWv}b_aU%SC z%%Grq4$LQsCcRu8)NClRR^9UjLXu}?m*{0Vzp-??lF0k=k5T2_y5`^9-NF12(9!84 zIm1_C=)e7HR_L;|IUj~5VI+inIK$)HI(@xNE7{(YpC*EMcV{tIsjp`uc~?N?mV82x z{SgU?xZLU=HSB8~G%mbJJ}ZXH`C_H8h|2Wd%(CzIvSl7Yu??+uuXK)8YK$n2cE?>;s3X>Xz(^)&XJ))x~mu5p=V(dlA zO}jTKoX?Dpn@~q%d)ZQ&eIeL~4hA^dBTX2PE09?lODa?IVw*arW#cS2;6 zXD>{3y^*{}O~Sd#S&b{cXId|lM)Cet>iwiZiB2sK9mBP`@f&yL`DUhe?>fOc4;ktU zB&_Cre&;eMnnddB~Lde{3?)`prt zITJax6K_Pb=j@F0*x$|`WV=OQI<3ktvhnFszu%)=*bY+`g=4du`j%r|ks%KgH;qh5 zs|x!-Lkx;u~i5(ls<`wLSI`si7po>~LZ#F)$BTbV7C-2=OFsXwAU z8WM2=4pwHZTgHk_;&_o9CE5=;5c$mR7(6wS?}j24}sgCAGvB+j|V7T)xVJ$zp1 z^cpr#!wOLHEQp2R4K`(F4!;tsV(2u60#um3TU965=- z`z@z7er;lUzT<_3JFB2~vSB|7iMnc1YhGUa!B4<$95S3NYl)?AE0OlwuYLKTrVeAd zHDr~N+sPTJA~9*^BFlGZMU?S zrMDG$o0(Hz0lS|1JKq5~rAy7gLz3=x71H5KGs1k7+g)PMqr%j=>3L^n8xVe_pzD^X zGZ*!OidS$M5bx7!Fo~w-;HcUKmA3A#{3y>T^Wfm2ZGQ8EwRdKO5g0x>RdU`yE|KcT z-FNA`tQ2fK-rBMdA=KkXyp3U3_}chi)Km)VO)J#rf0&n>MBW%Y5KG-8PmJE|wm;GP zoVj}qE>{-fjebvMm6D976bLuyIjX*i?^oI_h{5LcTVOb^Q*H(vO!EUPP;FZKWPHD$=;N8WjDIZl`fC8b{*DGC^Pd{mP;Qy~sz0!6 z(7q>L!Y;PrBg7nJyHK^|(g8xSb;9jMwOZT2hg#|eKQ|TQ8|&Rs836+eDev zr7XKh^qwZK;-|qG)2V8Dnl9Nh6}AhX@_VzjQuf_pr?aIW_kgMuxRrxG;x<=$=+-3-t*4Ujg!kbAhDG{{+YLXpC8k(}o;_3iwlm7ny5}dUA{rhB|sso(DLE}F8M$kuJJ#Cx~N!QSU%slSH2%}}JeS*E<@@bp8OvdZf3j=|9` zjHuS2I+RTq1Wt34y8D*4FMj1!(^4OpakfeeEp;9nl<4c7C(2*_m@y0ua5;W;b)fr%CA7P zht(s%=@O8FiVg1~*?+EQ0&)=$;REKs`ZH7(@e**SOK`di`cric|L@u|6&x?Z&yPvI zbHH$zYK|qCsu2?Sq#!0B!@rZf2ay~NUej%OW(f`i)2WI-M%}5OB_#QCorGr4J6(b{ z2BbxRf~GoOiUxi%wdRKcOxU_N9mV>R9e=w7gU;S}AJ)>6(|+p>mS{=9UaxPaFayg7znRmN9qafz+I@ z4cM_IU&CqsJ9BSrqpn{WD(kU8Z=dW zB#uVArIo!N!3a#sU9r4r5`uP9!B6(+S`bvnl(Oe5@HtG+=D)KoUpGhr{G ziD*4^3jw|u814f&T)GW_K(+8MckIA((F2JrdQJ>z2Id0>6GP<~Qh*=lL@&bcGoh~v zGf)R(-+ydg99QlFxHV!_Xti4SFp_b4eZ2r%!IL3iaCuz13*huPh7_>bn*dn-0o;oo z&=`v^#$`&{8479x6MwA!sEs=mK;Rb-1c;4s??m9(D>N*` z0)7`h4&bNbDj{gMk`84E2}9L-7(m}u$0q{RC8EnQCae%Z(;fg28*o@81SugE07BD5 z1KQ>RfF+*T4+YF>fe*!mFi>EA-jk^Vv_&-&^b$c}8V=it%K!<22u5V#i>+`oKxtKv z1%Dj7gvU;_YGjMRyE4c>V46v(JLhw31ex(j)-FOkOp*@qR z0+I?~L6vcg)4^ecF35zShExG2TP0$&k$=w)4)~77m0%;P0Gmkvx;HNk4lQ&+Awum! z4Q>O-pIQEvuU{S \ No newline at end of file + + + + +Created by potrace 1.11, written by Peter Selinger 2001-2013 + + + + + diff --git a/src/main/webapp/resources/images/favicondataverse.png b/src/main/webapp/resources/images/favicondataverse.png deleted file mode 100644 index 1b6d61ad1b4c2929ac8b7d985d25f6d740fc25b8..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 7165 zcmV=<5uKi000}DNklXnhZKXH)Lzi zbAXPgJz!~XK>$z!NcTNF`gtfC`)}CTdf55G}wm-7bfbZS=(0bsW?K_7c zATS5a=k@wO0btIVv5Ux#+JCU-K3lQk>L1X-0Jv zJ2&2S_W%fmx}`nFF8?Vc+`t4iGDhUr?^!=pjujs79d+;DHcc90PS7GL6KcuszF$)V zd8*8Y5Sh+cb!i&ll||v_B&vwBe^@K zfU9o5-O3KA@SGuovito9HaJ0tO9ovaPw&*yff@-S=D6e0t!t;I8;QXgNvEXo{&!Zb zZU6uVGXNI&bd<2q5BSg$+(q!6+wTiYmyNO&B1oyubsfX{ojZ39GTbxVKp31)>*B#( zRUsTYZo2c%95iS|wy0VZVYaTV?A);Wu1pGiUYW_KZFxxexYwxcx@}XpdKL>~b!yaW zQtIiRhnZsO#Sg99Mux^hvS~%$tshTQzs4(*S9XZ79FLc~za)x}RY!1G?mfXeZmHETx_Q9Ia5m~=L7g-U=_#xf8xgb9;gAcW`BDvc``i|uorRRAdI-_t~KoSW~(}e z=!64ChgpCS?(UXpvAlEb!<(x{R54#XJ$Zb~k5n?&f}Vd&BDcTr1zy){*EkCiKrNMm z*_3kVd|~zNW#|_yXOV`;3~H!KXSXg!L0veJg1c>^T({w|!w)++K+py!{zPfDKarOs zz!rws{1@$cp+O^n5IRRqz}u5)`z9eD{~$t0RzM;U05xJ*cRSsOCr9_K+qC4Fhossw z(|Cw^YNns-ZcZt%DL|^GA_-~3Yj3{pw4+X|eIz77*-QHf=spO(eM}20ZG*bZmm{qo z45jP5=;=^E>5&fVK4;jU;Colr002dTHz+#|apjG7#@>=qqamtV(rS3(vkrUh2`k6z zMp@OG-Gx?+#9jS>YPL*HC@7T2+gjd--}WbZN(T)tumZ~6Vr(Zcz~%9}?)WMAw1zN% zNREop2bLx!e&gQzi*ivmZJGqhMew`sSoMly4n5`gqZR`SBg~u)wK0qdSS2t*0DC+R zHC@0X5zv|-;d5&n0uW9M6%(iqnwg3KF7P&d@QI>fTfOcn?RU$($Z(yrsnpadcGnM^ z)f?97(w}PyOH-M#digUJfBK7GEW=x0zw%i<)WJrOKreby40Fn%>1eB^{P*|WclP=O zK8nbIWTU$~-GvDmuu%E{Cdq_azyX@Lfth<$+IM3k09by!MYrBQ7c{s;xWGFCz4oR% zI-Q9MS)6b)cZ6bRYPu^{oOki12fX`?mmYCYM`&KF8|z%sXNnln0FK8O4{UynO7ro$ zO)#X+ZvFuztVx zF2kQ*c**Tsc3E#soB>FR96fo8&T)y|%^N>{<~v?^_|jmNCLoni-&rzXkN!N)Gd&?< zd!iWBp4k8iFrWYmN4J4^=4Y?H;V2m{QDc`lI@Awdz6LTCF_!ie(oPLrgib;vxRVqSBfMQ{ zA_)mlV0Ijn7o7d><6R5$roh^e&iw2@uAi=aVtMu=dK9-5>MCSG2TtQ$7@RjYF-fAe z1c#F5o{}_~&7!7X1y-oX(s0GD9iMpn=`TEBQ4b^5b(#T&Q9t_kU;X0M*JJ5HQ!@$U zi;#zcc`3r_WN=}d3WmSDaQva)q}c- zTaG^O%E;ZK25Jd}t3wTJ-ziySS9j2xojz8>$KLdY6S}dR;n7E)IAeo8gO5hkP73k< znfed^?mBS0ry#MIrTq^N?Vs|aACsy?R-q=>e~ovA7S!EWJ$5C`VW zAgSagce}k|Gt<%fc}vGW{nlSSqyb=4QSTg#&iUKFpYBe~vP@C{YOGuF?n#;eBN_?8 zOg#&bNRt9ylIm&G;VS2ze(Fix&>tBrc@gU<`jHR)IU!VsW_s?H@r^Hj-h}Yl+wNAO z7KMpGf@>gyyYA(oTa+yz?$&>xjeL9jh=9_191^6ril3Yj1w2$y2f;UQ-8i{<%g-P6 ztbUo*n&q)L;n_d;jc%uoWsk&&$vY9tYa;o#WW>5u*9 zn~&+nF?ZDlQ7EdUe)vy6?4$`>Nfz*s@+rq2bNop!_~Fer4X39QkU@lwj|{OSvI9ao zd`@U2_xKTjbLIQ)X9}c)0(Z`dIk{VKk4=uLDm`g4vs`n(9vyx7AdkjNs0#a`@fb>g_m4<-K}}?fpzGcl-4B1rUjz(dDlfU_-=r-y=~YTK?XYM zLMWW#Lda-#Oo*A8mCt(C3s)ZVU!MCj2aomWxg;VaW$aYmHaYuOU;ECw$zifU*{#TP zSD#DjYvc~1npVgbb1~TUY~Z|io_=({=#xHZpq5KXphBaQ*$0HuBb?@LLI*&LHmp2W zTKwa!_y5VK|9(_1l5$a&m_rRhK%r}En!>#SQ@l+{S{tSWZ)e9X2JzfjQ>EGIl}9W) z`S{~r`Ml>H(~n~r;;>v<43%gNTz|)dU%KJ?o40LCi&ivsQL>vt1;XbZZ{CJI7t~tR zq6<7t9aZ{g&v@hE3|4xPEGCBu3Q2)&x?w%F{DAK3PdxsO$DQzu<$ZVx z)B?=OoDvipW=0p?c-z;mzjbRSmmW|T{b8!hpiHUoS`px((a=OXbFfremeiS*Bs~qo zT}L`T^;>WKnV=6ww%!Gdr;5%cp_ZARy3b z*$VMO4OVO5PWA#=WN-60Bh*A4gGEwr-Si8~mVWHlPd!3HaAa?qCp`hPO1wXgGVm^;5jQhMTgUL zLnT&D>{4|e7M$b?kygL&J4Zm^u~5&i#ii3^qJGlSW}JH)qg$d#s9o| zU8DrM!*W2PsHvLAGMSA}*$9oILIXhYKp_p)HB}_h01}Yp|k(MWcA+9 zes1+hLT@a_zG%_ZIfO`L1w$$_>(J+5cH20cmmPonJ750N(MUN~-%Gz;a}vQ7v^ue&%&XOvu^9!bf9 zx=D2!FUGR*vCXIb{0XO>@}i>-m`IIpbt!VC?un8BQo1mo-hRTNhyL}i{_=Qcr^8WG zK?PuhX(p5lqG?^=ZLeq8-4W>GS=`wvL)k|@Y1yK$p8H4ddf5w#sr5Zf!`#dabaR6{ z%nb>ufi}ef6fhU8Ay0aB+Ljt6pt8cJw^=V?fG*VmT{W&^uc-&mMNvoP7lt6R{;Ap$P7Q{;n8bw2Q zSJX``jn_q4g{og|e)+h5;!UquYE2!g z0um6Rgg}WB6hvs;7v(BHmic=ebH=%j9CtXf14!ylvx$xl>bg!jmfe(-yEzPIVy2-$ zV~dt7yYMUD7#7T#9VqChtbF!Cg?6@XI1&HeYT-ZgvTco&UH)G^?!Q(v`Z+tlk{b;^=x znujnSN!a3Y@s&UL`nRtMgXx+mLy@v|%Tn@8xoh@CM;`pAZ$Gt95Scl5q*!Q+#6kgM zj-w?UN!=jPRWJMQwchOyYts;{X$Trz4GK3zNyskE@}+-$`8WUeZEuWqMySTTeB(*S z3qWr)H#Li43|HOz@MpgKwMS>F-r{F4+0KkoJ*upI@F5xOdinF8|AEtAyVzZ&ayT`h z0y+y@h+6PsnzVW@ZJef=&H~@M^$rwWhk7%vD74 zb~Rz2;r7Xs@NV|Z8~}p@w>8FTZ)?ruvg#zM{8$X}7>x@>AU~y-c6YJ?ufE~7(CgQh=xE!?%v;LD zG1mw`EOK2oUNlrI2B!B-Pk;8R?_$N$Q4!$*1B)rZScwgTH92Q905R&ihu6|F;UVgb zL@x?!i6eJ1JF_pm6cTPIO=avHS_8*M7a-AVxWgixbps+_qF!^tJp)G0?EipL)5=PD{HVyKcx zT2iYsX@FTa_vDr-7JECgpZT#*fB6Tu%wjj^>+bNh@O?i&O;bzGnW0F;6*t{6D#m8( z1h|{E+B`=snzu!@Nk4~f3XWK_+VcDry-UyeA76R*ubuz4)5dmeDVixS>Y)rxT^j3F zLP$2F>L{npQ-c{`Tk^9$`sr=O*s2YiADL|$5OfOkN82Ly7hnoRkS!sxa;B5UwYgZ{`fu5?ZsYYzxk4t|Mcg7bdZcVwXut#jb?hW zBMlvHMD5mS7zW(|oc)P^xMRv^v_FaX{U6?w70x+gKP=qoE*NkIT$qV_AKUfNrmeO1 z;MBdEq+#9`7_AHIvI#pkAMMru`@Q#^f7-7s_f&Q@r9sE@%4NNOJ@1?|PI_LsX~Trs zV3Y^3Z`tXV+;i-#ovQxuZ!W!M^X&GqgKFs{jW^$QcQx1VL9y^t=;y+{Gyx#Hee-+Q z7ZZzflc)l-z1H)*HD9=^d)mI?lw%IN^nd-q>yA2naXspIYRu|>&ryv9Ut)IN>A(ER zv(N00hE1+#=*hD=%*mp>{M*;xw)T-~Z0S_&)1be7^T*%(;;)?Z*2DTC3K2R8WfbWE zS-OC9J@=FS%5g_r@)u{HxNO*;zB7ktwT&m?P*5%%#BmZ6?&7up8^yqq_bt=l|Y2Uia$8Qz4R4 zgSq`W>Y#xw9YeD|aah^;rw{zzdtd#UxcQOA7>%l3xgU18HChxF$0{_Vrp`9L?10O! z{=NZHb}S`bc+;LNRG*DL1O562q z9_h3sZDm8rpzm%Z0SPfQ^)lg-2LJQ5C;!9$JoA}(XVFa7&7fA+9*~2kLPMdUGi>~Z zTW+5Q5+N$u@=yB-FiAQBkic(VcY8u4*hokcjP5~0fnsLsGe_G$b@tmnfA*VK(E2{2 zRhpBEwA)tZxuyWf+`6+vkyF)!hYG)V#Ik?+i*sIfVZu&rUk^E$UztkP@mdy;xjtmz2HxN>%~Wo!3sB^ z43HoSGm3zFu}3!7GS>*3;9k&zX_>p;92DgXXP8DR@+|U{9bgd_KWK2PHc9c@= zU3u$WLn1@y`!f#DHFChOzWKiGxkT)UXSySeI?aqe_Q<9MQ3$1X$F8~k zuE!Fx1kd|9EIBblyW-YWHnA)Ln(Mx5oSHhw(`6rg@5kQy%JJ+f0tqJ3AaoG2`_;zbSQ zG{rqBfPGJ9mEeiYIoAgJ+BNG-=@!^V+to}x_Mu;Y&1D~W)ARZY6N$lLbo77@pdeKE z_&z!T1E?Kn(URuv0!Pb!6DlB}p$qB4wJ#pBo%##Uy5gh1fAUh6(+`o>^pCi8-DvHs znN2@wUYdL9u>)pcD9Lc?wYRb8#+|zkvHZ^;`M~eK;U$IT?6Z*FeJk*vq_vy>(PoL3 z7i<5)2hE?kD1QFK?>qY!PwqbUh}P4O?%ICOhDU21ds0=prMD>#%@T%=?X~@I-5R9f zYmYnbd!PQ`s}Eb&Z*pLTNMgR{;mO>mF}J8I2&k9j`rJ3a>Wk<6_F)`#Vspj!Zc5Yz zCzrmjVeX#WS7X+=&$xg7@~g*R`_8|*X7v{LoZW}DkA~*etk0U6KY7E<+6%S^*>ar+ z-pqPc50c+8sL%M+MJs>(%!eD#3%v4A1h{I_Ow-Jozvr9}-?nkb&TQGesfV6NW>v~o zC*L2#_8Q*gllc|gLu7pqlj9)%pONIN%o4hQ`-)oIAB4cmH*K`&b<4!%wUtfs{rW3 zn%$zN8MJJtMfUR+D{~_=K_Hk}uAAJ1PEQv&)BA3DXgPN`(BKX*yF-Gdc(cS?}r#odELi#xQqx40K~FYY%z_uO;- zFZba*+=u)7oyhF$%jKgYke!9l@&i=qP<1~005XW(h{oh<2L-Q2}FdK>Ro|kMKYYuYtax@2-dsB_tb8$?;+i z-};ScSn^K9!$jXsJQ&dVMf8cy_<;b5%*>k^kW;a=49D^;7bge1G_%>l8blMGMWMRZ zfZd3o$H}$qL8Em z5v0^sPzH%RIGcmGSh-liERvp9?i`dtXdnS+GYdXd38{aIfS(CcTDrPA^0BdbczCdS zaI!i$e`I6l<>h4qbFgu6u)tfexOmySLOfaQU8w$w_}?-l%w0^KtsGsg9PB}VWkO6G z+*}1IDdF{?e+vJL{=cN#yZjRbI2>$$YuMOX!EFDtxvQ1M|E2lgntwO{doiD~m8ZF_ zwuF_PxxEYg76d6dIKlrA_U~Gq|53~H{vVZp1@Xz)K|Y#mS(&+7{`;zbS4f*%eYAvk z+;x?yAjjYBg;@^e%dqt-^T+#R66R?%mm){gWzZ!5(v?s~ft7-n{>B5V= zSSbs4MZI7F@{=>3MURpdZTfIwe0nB7FEN2IsW6Y8quQ`$nOI>_$`}vt z%rP5QQp(NRYLXb`NZ~my^XO~O%rTDvKEynOTIok)v(O&#*Xyp?;th6v>>KF7oa^*M zaaN_jWfmA$y;=t51=-`B_x2Q`=S6OdJ%Ko^+^}wJ*Lu2 z9z>ud@|=HIarb^1_wKquz;U?h5YEsUlq@2&h?s)V6sH?>k>fzD1RQC2L1oNt!QMyw z7)C+5d8-s!4AoT51pw(W|7_Nxp7VZU?oSYnblGR_G0)OhO^*N0!835?$LXHI=DDr~ zGWn)YtvcL;>Rc+zXmToJQTdzX-c|lw7K1NXS12s-G_=d~{m;PpYE#EDpN03^r=Exu z0BQ@T7Ac+d^l#MM^>ewyGV0~jl&-#~-k;WDWaQ%bLu20{NNY<`K=3v>+Ep1LX=+v( z!{nK+cX#qvS;1-sY7G?SYE4byKdf{mODQ0bhCE7xoTS{Dp`Kl60a{jT1ym^GU|XS8 zYFQ`(8(9K}5(w zzd2s#{EipO(h@I|1o%ZYva_{Q?6fme(o;%!e<<1epKa`TxP9mpdcH*OtS)g?%+69? z5VF7ZXmv8}{}qI4v&$1BMOa`W-X*!kdoWx>v;&is?BCY0Gc6j<@eY{U7P`X@j?-k7 z(O{BQb*-NdO_C&emo~`kvQ+(g{pJN;a^p3r@bTMuVf(cKCvk;{_*Q?91Kc8??9CD_cIatm*{hyFek-D>UCoP@)g}`Su@~Ux#^U7%f z=j%1<(d&`6GN2K`j`~M(7C}GS51w8Y+G*r6xc3F)atO9K%S<<_kB!P`GxeC zv`^PHFbxtJqmfF%pkHjimv67LvVI$D(^_XjwP#d@6OR93cI`>)}|xuJ+Jt6g=T>@Nv{=iiz8 zc%BQfcl!oNCfztvzy*1ih%M$N#w%tb$>c!aR%+9; zhV^(EszG4o?HZHx#)#hcx|JV}Jfwvbfm zBVr_04m2#cnb>%NCSoPQECLW=M}8|ByDWHo`IFadTJ=yxy#bp*~@O%t1}NQ zrIqWas<5xR$(33;{B5B;PtgrQ+zT&1kr{nSzFAGeBuX^P8UmFL%iP{a6D?gTo^NN5 zr|5YPO-a@POyxm-uAh7@Nq6=pe{gm98Flm>3Zpb&3*;C{_?W?OIJX>)GxAgZ zX*ra}MwUaVEF(f^dnPo(EJ>f$f{lLkPK@FJ)9H;xD-$+h;A?h535gdXi7`#>QpJsw zqR~5vsMQ!_aub_0Z^RM!#1f>Z83}h_*>_3iCJh1#0V6q3uMWGZ2mk(G!g}7JcJ@E0 z6zKgrP3gH+&I7dGXY&ud9vv&r;Iq*GjZyjMOLMM3wHeTdO%n_N3LSHR&9N(r7>VxI*O}R`EW<*V{vA)+ zWINwTPV`-2)8uo)b{ojc%Eop0*aSDs#(~)VC#f}e9r!}XINeh_MN91sSAz#y8s^k} zB54APOJ_PKbG$7MsXHt7+zl8?!##&=Y`RTuGPusUNtV`$iF(0DQXXfps0(8Y|uY0h8}cFXnpUF7!G$Wd?(-hiUl!y8%@WwgZna13-H zWeyG?-*u;M8fU$OYd=MwSywIna?`EcjD3bg3v{tldEdRnI|y>V6aU(96^g18pW>~f z@q+|7iXo*n!loUUCeLt4&V53b$%prd5C^UX^W@jc^R$tB#3J!lu`W8A8VsTy8ttH) zbwdwdbNU45(SI~YFk_}N)gc)kAYnEo<{eIL3o^V(2K{Rv z^_T3gf4R;kM*rBoTUeKX9geW(f05~!R(=KGv^P3mRlm8IlnSk=f|R2p6xt+FzEqRF zAHDK3wTN2fA7-S^Z%z@|d5Q=EcbdB1#nDQsgzBCNYas7gH|K_+>Zc&$TPB{u3Tz1R z1v}iHOP29eGK1H$#3#*5?|3`Mw|}!|PuOOaanY>Vj)d3@*Ql#j(-9wMRr5dQO^S7n zl6qRYWK3_QL`uTfM%?5z;4&n09XP0{JiS$v9r-rfXW-SsvLNA3g%&0P%p#|=e_uN% zCAlK5D_&f>|vO^kwP^*RH%Xx*AS%}pxTb(jdmQyi7@&dDidvPut4 zK2!Ubt-)(luk4OUvEEkE+>0+oEq<)yX)Usqstu+qgabimL%6v+{pr@BXa8kjms#Zv zexbGps0YWJEKkx`K6rvmJXy7+Gynq$3lSEGWFOfkdOTePAsK$E>o^?7)Pah8P3&_Y z80X>f`4h(D0FJ^@m7_7avL%RDk5(0leWnk&F|vC+sUyYm=>O|+YElTMl0R-cf4W@h-e}>l!5e!uai7*@&Om8Pw5)!SbidjCQwy^_|Es5+=bK+ zVHD`nIGhO*z6>r39pDciKMc(C&mmd#sjcpPjwT%--{ujcW@eR>v1FF+tZJ~-f!yH) z622}<3|C}?oI@|tvg}N^shYAKv%Wjo<}w2+n1_}w9*0jBo9+v;RWgL783^<>@eyDm zILfjr7fcfFhy(&JaI}6LcFvmuB_9kEDWUYr5;&3<<1>Z~!}=PTG(^=a<*XN>w<$hS zS$T1eLNUJ)U!fQ0sk|aQHybZt?rUb8e-Sy9l8pjs!vBtAx+(!+=DZft%hIO=eA-~3 z_{dKSEj8*gVRtEfahQGSrZtO?nj7!V;Z5Mh^@uOOx>yb+y(2Y;EEcNa4x+wvf(`VfY^r<#20q<|Cr>)b&67#s#b zH8pp)&7EIF#HGZK*+oPH0;seTD;~Yv*5X}qlX4KwpcuNZNKU{~bgz>5Ogl=nq!sKt zM~P0uc&F!^J5X2@gVRkX|dnBtFa~P9;AmM z1tQYaGH0N9(z+(^q510EpmaG{=$LO};Qs)@pktnhvGwi&?>R;C|^1pkq4Zt8t;}gXZ$D`$VCpUVz60V z^5n)cJ>4iUM!2p?;?Z940p zIpDtK?nxBd5%wskL3)lmt9@Ms_yypld<;YJwQUvW@hOn?vwRkZsFQQ;RDHc9QzWP0 zk5jcpeyOA50!-R-&IE*9k3RPIHkm?UaRfcR)Ud;+$OEItyYzKxOsR0(TmUtq3W%%A zaFF?9bfB8ZyU*aY`C3ZShTx&hB{lqgN2o(c*|S02vo^LkStDKmzF%e zIHm17G@axLvHJD3&>`L6GPh;!7|tNxE;{yv6k}9of$V+vxTOIY8(w zr$sejDb2TBkr zLDEdhR@>?H^nomAk}A&UwHo|D2}$_cY?~-y9vqI1!No`njFn(Or2aV1T<_~7HQ`|$ z@m!s^7Nmg=2vpt<Q&eDC==mA_7g&!c_$#-FPsL@zofmN;p)`Zz67|WfsnnDiSAD zGVaG38$1-rIp)Jj9TzXF;!saI}b%30V8ZPMmwzj@3R`H%CkLTs zk3^sZ0)R!@;CHCRG=ML>>6Cy7uxy;ner-IFIq6WmtBx*Akrsy;BpP!9<`*K#v`wpv z;%UE94$rB#TGL6Gp`ao{7707lk(CAADUB6yOy zNtq-^N<%%VpoH7RRTX0ayPP|}RVMLr8lSf(ddJm)?8ExHU+(Q&&o;8hwe;>M>WBDe zr|7p?`7bw>=<%v;&^Xa9_WK*ieXHwd$=AUl;b)P`5eW3yP%ttgKE1IBM(DcP(YvJE z$Qn$N*CA2n1wXrSYJ8R+bo-SL5_<`OC{T|mY%$QBLkRJyP7~V9th72QqVAWHCrLk@ zLH+*FoJ8ig&uWWU9V_hkPL{}BgkJ-?j?)~y4?A+{h+mN#76^!!w|V&ax%J5u&T(H~ zUZ*MI?<7^J_*Yg*u;9HANu|r+3u`U1!@RY+681|sKdVHc@N9M!EX{~8n3YQwUg^>c9jFY z(xsgG)6=g|hZ867Q|wSSO|E>H%FJB2yyw+OHR8&(p3mZYZ$GDhXgJPszo!RK0DytO zjs)Nk5CO^nD3eiYE8-AN5ADSFCmZT#`%LKK!dC?Elht3S ziY24(ZA(TphI#GOs=q=J^{a*2FD)w0L zKk2=kN3|YzdblT3I(%9!$T{+{Jns-mon(T=8nm1AT7`z8Cl+X})OI}Gb^7sGItg~x zwlWGbK;9psQq&buAN-tU?%eBD8ns_(KRF@u@bs@~p;ah1jy>t6N%Z~%DQ9zB*9|U4 z@MVv5Fyi>iq41m$dqtL6bJvv#s>c;$e~B1~0AIAYFbd751lCVSICqh*zRmO8I!^vT zXpK}w1HQWL14;t1`&5$$m6>KXVbg`Q4^AIGbdJ8MPD21DQ4tnW3w*12?{bW$F~VC- zOv7j%>7sVh@~wfjAKdskBA9TemY*-ggJkt)#eTuh#Xg_(XlQY(r$o+qoL0=O*d) zboVDS`b{_72jL>2V=Enq&U$41%DT$1^!(!(9xB7rTNK`Z-CSK&SPc@?p^1DSRm|0c z<3NA-3ffX^_iRtTw8XiM9@@*?{r2PcoVQ;sz`DsPch?76g7pRG(e4|C!ymAXWf?>R zKQ03S(7^G1+c(@qG}aOr7U2BEg-3+0Gaywi5F44=vlTniDMr%^BHGt0Ndo)4U|5bW znS}$S01)O)>o5rG37oX`mZw6TFZS>ByOC*hwA6{{$3!)7GPsGEeCAA+P%!#tndyhd z%X94a=ce8*3EH(fY*aljwl#7|=2WTdZmLFoKgLqzZ-2B1>%{{lS{JTQ)1*Dudj3wI zU_P_5V0asY@&I?J+~jq`DylxF^I6x%Pu2}-f4MHZ4bkD7hRo`!Q=&`r;gMLLVa+|m*HMapkT}=WkPF@i21*fjkpAu~YQ;Y!(*8YACYk8GN z_q5DVRU`lseT*~^A4I&pvPZ0|m1(6s1Qv5klQEZBu=@?bnX~TUK(cy$OK|SYHlV~N ztvou!hvTD(kICescC63Ersbk}muyI3kZKpM!lXf$Fg=%Vmeq^LQ!S8E5uZZrx@C3u ziQ3LEpHGy6pWR_pZmRG7MxnaNFVr|jIR)+XoCX)i2<0K;i|Lkls%$oL0?0=QB|X&bO~bHZO?Sua6&?8q_;=_7?2 zy=iW|p4h>P(r6#IQN!jr)C<3b$B=%uku-0x|U8nbF8j1iR>N(jLiyH5mb*FU{Yei#X z75Wn*C5Q0-h>4Z*W505y8&VVpFbBh!nQ+*POJ23Spa{6(;`l9rfrm+Y#5tey`}a7n zxj}r&-6-rlW)Mmb>%iW@K#I252^@sf@ni*I=Y#NSdA;3KYFLG3sDK!B9c=x0A3;5M z`Nmn`~CIKa(6uflYAThM^1)qrs@JvdxL4YluyQ!+G zvN{RK{)%#PdlA@xEd?=)k8(2Zq&GB2#Lpg^}e&$#k`@m z_l5PQLqlp2RNIU{NC3q26hJBoZ0pyV3!E0#BMJmZWGrmMyb6wl3uL=096>9UWNziq7esAl6w~{&+%lC zlda#vRp@8^MHG(!S>CvBh6s;p42xWOn>pmO6r&jMA2M)jp5WmR%OTQrY}`*a^e-iV zfS5T0c4gT>)tZk}bpc(f*1z237mpt;l@_5SlJZrblBF0V5$uzPYoa!pnM0f+#IKsB z-r2!Dq7q}vUYrxgqEFI*K>8n_%9CRF+rJ(jq(sckUSZAo2F-6uy{;caL^KND7@Q|W z%*Subb3CPIikIkN?4c|cCiX#(`l8Q_=H#17hsusj_z5ybx^70PNo!HCEdY1xiE67g zJt!u=Z=;_=8ENv7LNtG>+{ID-zEmU?{anH;^YwWb)i;cI^6b)+3lx&5 zBwQS&5+-bX3*o3Q@;y>)$5>8J;13Lvz-r>*-a{NO<0mkYzykup_ha-CL(E8$inwr| z?51D}2A`6~kd-^}Oj$<0qWS|23iv<@K}Z^XpsR?usO$Oofe6_7SlFL?nwwta?dW+! zMf6gUZKj@SB&%nl1Q^7u?WcDM`!2mc;F=hs(HdnV zu5NC;ieBlcP^Iq~#JijE->AwfbzounDjhooJxH$+>@0UN&>2gQXUK|e5V<;xE%0%;TR1H z2<)Y260NUdE`|rUM-r5LJ_zp;Z=*^A6LD3%%AVpQmI`%TsFeoFEB)*WidO8kD?T&S zp580?WIka7z{Cfe>!A;If4ClnuP`^)`y_oKmx>eROD<`p?D*4hR;)&uN-|-cRt5Wh*&5QnslSmKtp*9wckc5FL1oC zH8l(6f9z3-ij$&cFWGuo?<=?BncyULQ;{6;2!V&UpC}^L=1i4`@d!B5!bfG41AYsi zC2%*6Hu(a+`5CWwM7P>@h@@XB$E=BGEN0Jnufmp>-+wilGrhVYzqMW2dE7|R*jkxU zX0=GmwpzsbL6&n7<+s3TtvsBN$RW0O;B)Wb=XDOZ>~=g60r2i?=}~R*zLMAW|Fhiy z4@M8E;dQjO{kdD9%-}xl**SVrCI(2^9tYH4>a`)jB&h2iRtvWtgT=dckV4b}yPOJd zlx%ID@5jc{=C{N>#a=1u3hlkcBV_JxNIM^gKK?dNr7EXSes@QNF_iWu#IOyjzGW$F+>t=70&rS872fPGoETiU`u`ZhJ=K@{uvy7m6p}H z*v2MaK(8e8E7M}VM3%9mjW0ml{4L;+B<)M&oKb}V5P%`|-bi@_TrOAR-&FJ3B7@lk zEVkPbdEnr?RzTNCe)AMj1}9TK00kXyIUa2Z6UPrdiIsZxvGV4&b3*LzQo>xvo=Ib#U7mXwCcjoM|wU952gCi z`W$-l``p3#MP!%`R5e=u&#;FF#-e@~E^Y72Wfr;NkW3O;h|ABp*#kKm=5D+iQ^$x6 z88@kkm2`dwZEZ3?Y@`xO!84!Xa{BMwDaiyY*AopvpPw7;*ZP9HuR9uDG@L?v@jR_O zl**)>&AH2*2 z0HB}#%>~HHU~4IA3G59=q=9q%o?^g&@Zp!d{=fF`#yc}SV&zYWv)|{ate2#ko0-5D zaCXF(RBJBk;rL}?J+{PDt{Mt!@QxTQ&0luUHXf033VUDv!B;T*M3pFXc8k!i>fFEo zx?OdK3rrEkgcD4FJ03W{cY)$1_KxF>eK!9){R99ep&wW1ch!gP3pox4^|815sLc*k zViJsBLa;$_N<6NDSBwLf{mTcujVN{_v$8Y*vV#9T14GjDdE-DQrp(td7 zGeW^h8jrO1C)`K3g+3P>$6xq0fByW%xTE@rcD}h$H4avKHN6{ygqkmn#)~Kd!l^!2 zy|}J2aD+G&f%1gCb;QlB*H`pfQ0!P^=)w`j@KvDLP~Hi4J3olhNe3!ADAnm+_k#km zi*CJOPlAF$_ssmDpm_0}x8G-w)v_DpXr|*#OV_7!SsNzkR)EKK`O|v($fkxQ z$%zLBoNr95(kQaVKiJuN@cTFm-ytxQZspEZBh_5@6Gl049OYKD&5+RM4L6k+QgQ|N%qG506AA}(mtDf<$4d<1UcZmbe0K^`;G40l6&L*uo2_Cfd98l?Q3f4CeUcYNZZ4%F{(NSxm7 zKPlb&vOXWIqm`CC$8Pm5{Y{rYoQyZi^gT^z@W6WYdq&&8_W=NbAIl(?P5W{C^&+lXB*)@5CCt_azSA*&nYIHy$<<)a#xtzKz&L_RsLwJhX z6zB3yhuL4!gc3Hui?{%6q*Sz8h)m7Z2c|OCuLFWDf2gD^WfWx_6fae6PB>dzM&AmH z1=6Fb_)qLbn^7u@1U6tPM)3-yE$|MD#(f(J9cm-5D9e6RAx$5%{tD`UH{*^vywuw9 zl(rolu_PCE6LQD~?>#t|p`wkrqWhAD* zp{{dc!AI0e^9+@VS%JTug^II0g2RMr7EYo`F&<~uydx}()alFBT!QO*>ayP!C#lj_ zUd8Y4{6|t+AlGMcywk$&^OT=hOEqREeb;Po7?voU>H}6J@}llFL9k5qh8niW2z2zMqU!mwlMNhZ2(TSqF-p zcab-02j6l7nriN5LjUwVT)=^WQ>FR%?pQyg7W7y2Ili(kD#Mdq)Z)Fy+cK}5BEMUT zlVOSACDnw?hlW0=!!r%m$+RN-17`V0k41LxLVBxJ%B)`Z!s){ucUM%6P~s?D{0Zu$ zL;RHVWq#}Z?1tsK5?d#GYl8-eB&2U^8aYAng#Ui0A~Y};5Do`Co)f4rrg?{58zn{i zBx`O6x4&|Hy!(#c0z8{xVX93QqHFPOQ!TdL3loR$XwpXeN^vrn?#eN;_8a;lNK#I) z$Q%38btEvP0u^8Pc`fMA-~l_wX96z#k#dv6JEy@Xu_-L(iV)vniIMcKsLGYcA*J8T zXQJ2Apb>;LmNUV{-RqW62OWLd>;jW}GK!;(t&eTE47136)R|HqJT*oyfBM~s_WvR{ zN6%hzH};b}h9|HtPPYvZh6MHm?%6Zz90MKrFjhoV9*~Cw? zyvHBUNGy;VwX3fWW#XJnT`vO%yY4U|!mC&t^c0ajf5dM|42=bha@A=KzWf zIt~c&d!@ub9!$fzzGLD{=euBe1_DEgD9wgzq(Y>Qc2E9vQmW zVyekHadtPl27@g%wrd8o<(uPLO+4I}8%rhUIN3Ce5|ZGyfx;{!^juC27w+ghP8{36q-xi{TTO~KDjmW)r`Dmt za&G%&wY7JbL74ImOA$fDKYn{5y-RzGfnU^5N5lUlYMFdp7Nf;AT6J~F!$7;$W9mY* z|8$~_3o`#5BMY}6A$aFPtgTz%HYY`hw@ZlGJPk9+YsE;Xot|!MB$*r)dVAd(nzJxo z@8{_34&fI8_F%aD3_(BE=Y7Vg;NsTbRSESR zrq|CF{ers83@TjeU4|?!@7To#8NDF!nx_zW$R}YjrAkBc!)ny<4!TRTcfD{=^uluK z>t38bG!GD${WW#@mPqN~QE^z2=fG5Hel|G!7kewercGjve1oB&iOXjaTJd5GXtPSj z2kqA~T^tP7Ug~F!@VvonrD=MwSfXkMo8{RJ36w|7<#K;6i3v52w(%#?dd=m&Xrvkg zjwI2|n;ITeSQ=lFF-w}Ow>Vjj3kT=;o@j$ZpVOJqlc2Fsp%2Ukzf-#r{(mCF7EQ1B z)mj1+QsoD?$-h5(xGg>+AgBWKrb9@+`adFG>VNGfE`B!T6D*qJ0z?QG?y&DEoKuogU3k? z6S5HD5yqu<_cbal7Ntr0G0BXwNj#$_$aEuN((6PHBlSF9)_TF`G-3PIeDd>tYrJP@SzF?0WaOKnGYZoUAKH z!;H`_Y9o-hgF;-b|u*Z)uPuf*F)-n?h5 ZpMtBThx_qz@U&onjHIGOm6-9T{{==`;wb Date: Tue, 7 Jan 2020 17:36:06 -0500 Subject: [PATCH 082/157] docs --- .../source/developers/big-data-support.rst | 20 ++++++++++++++++++- .../source/installation/config.rst | 3 ++- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index bb16dd9133d..454fbf40f84 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -6,7 +6,25 @@ Big data support is highly experimental. Eventually this content will move to th .. contents:: |toctitle| :local: -Various components need to be installed and configured for big data support. +Various components need to be installed and/or configured for big data support. + +S3 Direct Upload and Download +----------------------------- + +A lightweight option for supporting file sizes beyond a few gigabytes - a size that can cause performance issues when uploaded through the Dataverse server itself - is to configure an S3 store to provide direct upload and download via 'pre-signed URLs'. When these options are configured, file uploads and downloads are made directly to and from a configured S3 store using secure (https) connections that enforce Dataverse's access controls. (The upload and download URLs are signed with a unique key that only allows access for a short time period and Dataverse will only generate such a URL if the user has permission to upload/download the specific file in question.) + +This option can handle files >40GB and could be appropriate for files up to a TB. Other options can scale farther, but this option has the advantages that it is simple to configure and does not require any user training - uploads and downloads are done via the same interface as normal uploads to Dataverse. + +To configure these options, an administrator must set two JVM options for the Dataverse server using the same process as for other configuration options: + +``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` +``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` + +With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific dataverses) while configuring only direct download, or no direct access for another store. + +At present, one drawback for direct-upload is that files are not 'ingested', e.g. zip files are not unzipped, derived files and metadata are not created, etc. This could be appropriate for large files. There are plans to remove this limitation in the future (potentially up to a configurable size limit so that the largest files retain as much performance as possible.) + +One additional step that is required to enable direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. Data Capture Module (DCM) ------------------------- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index e7291b81034..8915bab89e5 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -499,7 +499,8 @@ JVM Option Value Description dataverse.files.storage-driver-id Enable as the default storage driver. ``file`` dataverse.files..bucket-name The bucket name. See above. (none) dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` -dataverse.files..url-expiration-minutes If direct downloads: time until links expire. Optional. 60 +dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` +dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` dataverse.files..path-style-access ``true``/``false`` Use path style buckets instead of subdomains. Optional. ``false`` From f6af668b700ae8505562423c68accfa29ed88f2c Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 7 Jan 2020 18:08:15 -0500 Subject: [PATCH 083/157] pom version --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 77910698319..481f0418c3c 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ --> edu.harvard.iq dataverse - 4.18.1-tdl-dev-ms + 4.18.1 war dataverse From 86d7ee6a9378230b0f656bd6b0e410766cb2db11 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 8 Jan 2020 11:59:20 -0500 Subject: [PATCH 084/157] add tabular and fits metadata processing to direct uploads with size limit --- .../iq/dataverse/dataaccess/StorageIO.java | 9 ++ .../dataverse/ingest/IngestServiceBean.java | 97 +++++++++++-------- 2 files changed, 65 insertions(+), 41 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java index 9e0cf7e11b8..2f66eec5f4c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/StorageIO.java @@ -533,4 +533,13 @@ protected boolean isWriteAccessRequested(DataAccessOption... options) throws IOE // By default, we open the file in read mode: return false; } + + public boolean isBelowIngestSizeLimit() { + long limit = Long.parseLong(System.getProperty("dataverse.files." + this.driverId + ".ingestsizelimit", "-1")); + if(limit>0 && getSize()>limit) { + return false; + } else { + return true; + } + } } diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 35d8a5b4dd3..f8612a0cdf3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -72,6 +72,7 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.io.InputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.nio.channels.FileChannel; @@ -178,8 +179,9 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List saveAndAddFilesToDataset(DatasetVersion version, List dataAccess = DataAccess.getStorageIO(dataFile); @@ -355,7 +324,48 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List sio = dataFile.getStorageIO(); + sio.open(DataAccessOption.READ_ACCESS); + tempFileInputStream = sio.getInputStream(); + } else { + try { + tempFileInputStream = new FileInputStream(new File(tempFileLocation)); + } catch (FileNotFoundException notfoundEx) { + throw new IOException("Could not open temp file "+tempFileLocation); + } } // Locate metadata extraction plugin for the file format by looking From b373a723b20587ad17ae4d08bd3a951b217318f2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 8 Jan 2020 11:59:30 -0500 Subject: [PATCH 085/157] docs update --- doc/sphinx-guides/source/developers/big-data-support.rst | 6 +++++- doc/sphinx-guides/source/installation/config.rst | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 454fbf40f84..f1371f41759 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -20,9 +20,13 @@ To configure these options, an administrator must set two JVM options for the Da ``./asadmin create-jvm-options "-Ddataverse.files..download-redirect=true"`` ``./asadmin create-jvm-options "-Ddataverse.files..upload-redirect=true"`` + With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific dataverses) while configuring only direct download, or no direct access for another store. -At present, one drawback for direct-upload is that files are not 'ingested', e.g. zip files are not unzipped, derived files and metadata are not created, etc. This could be appropriate for large files. There are plans to remove this limitation in the future (potentially up to a configurable size limit so that the largest files retain as much performance as possible.) +At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by Dataverse from the S3 store). A store using direct uplod can be configured to disable all ingest processing for files above a given size limit: + +``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="`` + One additional step that is required to enable direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 8915bab89e5..bb8009b3366 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -500,6 +500,7 @@ dataverse.files.storage-driver-id Enable as dataverse.files..bucket-name The bucket name. See above. (none) dataverse.files..download-redirect ``true``/``false`` Enable direct download or proxy through Dataverse. ``false`` dataverse.files..upload-redirect ``true``/``false`` Enable direct upload of files added to a dataset to the S3 store. ``false`` +dataverse.files..ingestsizelimit Maximum size of directupload files that should be ingested (none) dataverse.files..url-expiration-minutes If direct uploads/downloads: time until links expire. Optional. 60 dataverse.files..custom-endpoint-url Use custom S3 endpoint. Needs URL either with or without protocol. (none) dataverse.files..custom-endpoint-region Only used when using custom endpoint. Optional. ``dataverse`` From 93de5ce5b62b88bb39caaa8c93407e1b1b50ced3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 8 Jan 2020 12:39:27 -0500 Subject: [PATCH 086/157] fix tests --- .../harvard/iq/dataverse/dataaccess/DataAccess.java | 11 ++++++++--- .../harvard/iq/dataverse/dataaccess/FileAccessIO.java | 4 ++-- .../iq/dataverse/dataaccess/StorageIOTest.java | 6 +++--- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 6d7fc48e846..36b1a4a5204 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -126,20 +126,25 @@ public static String getStorarageIdFromLocation(String location) { } public static String getDriverType(String driverId) { - return System.getProperty("dataverse.files." + driverId + ".type", "file"); + return System.getProperty("dataverse.files." + driverId + ".type"); } // createDataAccessObject() methods create a *new*, empty DataAccess objects, // for saving new, not yet saved datafiles. public static StorageIO createNewStorageIO(T dvObject, String storageTag) throws IOException { - + if (dvObject == null + || dvObject.getDataverseContext()==null + || storageTag == null + || storageTag.isEmpty()) { + throw new IOException("getDataAccessObject: null or invalid datafile."); + } return createNewStorageIO(dvObject, storageTag, dvObject.getDataverseContext().getStorageDriverId()); } public static StorageIO createNewStorageIO(T dvObject, String storageTag, String storageDriverId) throws IOException { if (dvObject == null || storageTag == null - || storageTag.isEmpty()) { + || storageTag.isEmpty()) { throw new IOException("getDataAccessObject: null or invalid datafile."); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java index fb065dcda3d..7a8231da6bd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java @@ -50,8 +50,8 @@ public class FileAccessIO extends StorageIO { public FileAccessIO() { - //Partially functional StorageIO object - constructor only for testing - super(); + //Constructor only for testing + super(null, null, null); } public FileAccessIO(T dvObject, DataAccessRequest req, String driverId ) { diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java index 4d2e1072950..513d4d90988 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/StorageIOTest.java @@ -77,9 +77,9 @@ public void testGetDvObject() { } catch (ClassCastException ex) { assertEquals(ex.getMessage(), "edu.harvard.iq.dataverse.Dataset cannot be cast to edu.harvard.iq.dataverse.Dataverse"); } - String dummyDriverId="dummy"; - assertEquals(new DataFile(), new FileAccessIO<>(new DataFile(), null, dummyDriverId).getDataFile()); - assertEquals(new Dataverse(), new FileAccessIO<>(new Dataverse(), null, dummyDriverId).getDataverse()); + // null driver defaults to 'file' + assertEquals(new DataFile(), new FileAccessIO<>(new DataFile(), null, null).getDataFile()); + assertEquals(new Dataverse(), new FileAccessIO<>(new Dataverse(), null, null).getDataverse()); } @Test From 0cda1d07d6067186eb362858030c72ed4b4566b2 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 8 Jan 2020 13:06:49 -0500 Subject: [PATCH 087/157] test fixes --- .../edu/harvard/iq/dataverse/dataaccess/DataAccess.java | 2 +- .../edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java | 4 ++++ .../edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 36b1a4a5204..887558b4dc5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -126,7 +126,7 @@ public static String getStorarageIdFromLocation(String location) { } public static String getDriverType(String driverId) { - return System.getProperty("dataverse.files." + driverId + ".type"); + return System.getProperty("dataverse.files." + driverId + ".type", "Undefined"); } // createDataAccessObject() methods create a *new*, empty DataAccess objects, diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java index a1a96a363ed..c3fe324c641 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java @@ -2,6 +2,7 @@ import edu.harvard.iq.dataverse.DataFile; import edu.harvard.iq.dataverse.Dataset; +import edu.harvard.iq.dataverse.Dataverse; import edu.harvard.iq.dataverse.mocks.MocksFactory; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -12,12 +13,15 @@ public class DataAccessTest { DataFile datafile; Dataset dataset; + Dataverse dataverse; @BeforeEach void setUp() { datafile = MocksFactory.makeDataFile(); dataset = MocksFactory.makeDataset(); + dataverse = MocksFactory.makeDataverse(); datafile.setOwner(dataset); + dataset.setOwner(dataverse); } @Test diff --git a/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java index c72bc5d6faa..bb19a32c4f8 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataset/DatasetUtilTest.java @@ -29,7 +29,8 @@ public void testGetThumbnailCandidates() { DataFile dataFile = MocksFactory.makeDataFile(); dataFile.setContentType("image/"); dataFile.setOwner(dataset); - dataFile.setStorageIdentifier("file://src/test/resources/images/coffeeshop.png"); + System.setProperty("dataverse.files.testfile.type", "file"); + dataFile.setStorageIdentifier("testfile://src/test/resources/images/coffeeshop.png"); System.out.println(ImageThumbConverter.isThumbnailAvailable(dataFile)); DatasetVersion version = dataset.getCreateVersion(); @@ -45,7 +46,8 @@ public void testGetThumbnailNullDataset() { assertNull(DatasetUtil.getThumbnail(null, null)); Dataset dataset = MocksFactory.makeDataset(); - dataset.setStorageIdentifier("file://"); + System.setProperty("dataverse.files.testfile.type", "file"); + dataset.setStorageIdentifier("testfile://"); dataset.setUseGenericThumbnail(true); assertNull(DatasetUtil.getThumbnail(dataset)); From 14c71344174e363fca5ef68dd25dadaea908bf99 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 8 Jan 2020 13:16:49 -0500 Subject: [PATCH 088/157] define 'file' store type for default test --- .../java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java b/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java index c3fe324c641..5c148c8b77f 100644 --- a/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/dataaccess/DataAccessTest.java @@ -54,6 +54,7 @@ void testCreateNewStorageIO_throwsOnUnsupported() { @Test void testCreateNewStorageIO_createsFileAccessIObyDefault() throws IOException { + System.setProperty("dataverse.files.file.type", "file"); StorageIO storageIo = DataAccess.createNewStorageIO(dataset, "valid-tag"); assertTrue(storageIo.getClass().equals(FileAccessIO.class)); } From d41b27b6a33acb5e45848c8c60405b01e4db3603 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 8 Jan 2020 14:21:49 -0500 Subject: [PATCH 089/157] broken logic for normal upload --- .../datasetutility/AddReplaceFileHelper.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 4346eacb22f..6716cb15327 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -950,15 +950,11 @@ private boolean step_020_loadNewFile(String fileName, String fileContentType, St if (storageIdentifier == null) { this.addErrorSevere(getBundleErr("file_upload_failed")); return false; - } + } else { newStorageIdentifier = storageIdentifier; - - } else { - this.addErrorSevere(getBundleErr("file_identification_failed")); - return false; - - } - + } + } + newFileName = fileName; newFileContentType = fileContentType; From 588dec8816bb2f464412c3204e35cac63f16f68b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 30 Jan 2020 16:58:21 -0500 Subject: [PATCH 090/157] fix create mode Need the dataset PID to be assigned to be able to provide a direct S3 URL. --- .../edu/harvard/iq/dataverse/EditDatafilesPage.java | 11 +++++++++++ .../engine/command/impl/CreateNewDatasetCommand.java | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index ce2d8383408..b1af137a938 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -18,6 +18,7 @@ import edu.harvard.iq.dataverse.datacapturemodule.ScriptRequestResponse; import edu.harvard.iq.dataverse.dataset.DatasetThumbnail; import edu.harvard.iq.dataverse.engine.command.Command; +import edu.harvard.iq.dataverse.engine.command.CommandContext; import edu.harvard.iq.dataverse.engine.command.exception.CommandException; import edu.harvard.iq.dataverse.engine.command.exception.IllegalCommandException; import edu.harvard.iq.dataverse.engine.command.impl.DeleteDataFileCommand; @@ -36,6 +37,8 @@ import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.EjbUtil; import static edu.harvard.iq.dataverse.util.JsfHelper.JH; +import static edu.harvard.iq.dataverse.util.StringUtil.isEmpty; + import java.io.File; import java.io.FileOutputStream; import java.io.IOException; @@ -1748,6 +1751,14 @@ public String getRsyncScriptFilename() { } public void requestDirectUploadUrl() { + + //Need to assign an identifier at this point if direct upload is used. + if ( isEmpty(dataset.getIdentifier()) ) { + CommandContext ctxt = commandEngine.getContext(); + GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(ctxt); + dataset.setIdentifier(ctxt.datasets().generateDatasetIdentifier(dataset, idServiceBean)); + } + S3AccessIO s3io = FileUtil.getS3AccessForDirectUpload(dataset); if(s3io == null) { FacesContext.getCurrentInstance().addMessage(uploadComponentId, new FacesMessage(FacesMessage.SEVERITY_ERROR, BundleUtil.getStringFromBundle("dataset.file.uploadWarning"), "Direct upload not supported for this dataset")); diff --git a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java index 3ce10e40abe..e97eeb47ab3 100644 --- a/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java +++ b/src/main/java/edu/harvard/iq/dataverse/engine/command/impl/CreateNewDatasetCommand.java @@ -69,7 +69,7 @@ public CreateNewDatasetCommand(Dataset theDataset, DataverseRequest aRequest, bo protected void additionalParameterTests(CommandContext ctxt) throws CommandException { if ( nonEmpty(getDataset().getIdentifier()) ) { GlobalIdServiceBean idServiceBean = GlobalIdServiceBean.getBean(getDataset().getProtocol(), ctxt); - if ( ctxt.datasets().isIdentifierUnique(getDataset().getIdentifier(), getDataset(), idServiceBean) ) { + if ( !ctxt.datasets().isIdentifierUnique(getDataset().getIdentifier(), getDataset(), idServiceBean) ) { throw new IllegalCommandException(String.format("Dataset with identifier '%s', protocol '%s' and authority '%s' already exists", getDataset().getIdentifier(), getDataset().getProtocol(), getDataset().getAuthority()), this); From 0b59f5e38cfa3f432da9ad0b7d7ed9857288296c Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Tue, 4 Feb 2020 14:19:13 -0500 Subject: [PATCH 091/157] export --- .../dataverse/export/ddi/DdiExportUtil.java | 94 +++++++++++++++++-- .../dataverse/export/ddi/dataset-finch1.xml | 19 ++-- 2 files changed, 96 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index 51975ba5efe..d35b3702d29 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -71,6 +71,10 @@ public class DdiExportUtil { private static final Logger logger = Logger.getLogger(DdiExportUtil.class.getCanonicalName()); + public static final String NOTE_TYPE_TERMS_OF_USE = "DVN:TOU"; + public static final String NOTE_SUBJECT_TERMS_OF_USE = "Terms Of Use"; + + public static final String LEVEL_DV = "dv"; @EJB VariableServiceBean variableService; @@ -173,7 +177,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeStartElement("citation"); xmlw.writeStartElement("titlStmt"); - writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title)); + writeFullElement(xmlw, "titl", dto2Primitive(version, DatasetFieldConstant.title)); writeFullElement(xmlw, "subTitl", dto2Primitive(version, DatasetFieldConstant.subTitle)); writeFullElement(xmlw, "altTitl", dto2Primitive(version, DatasetFieldConstant.alternativeTitle)); @@ -188,10 +192,16 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeProducersElement(xmlw, version); xmlw.writeStartElement("distStmt"); - writeFullElement(xmlw, "distrbtr", datasetDto.getPublisher()); + writeDistributorsElement(xmlw, version); + writeContactsElement(xmlw, version); writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); + writeFullElement(xmlw, "depositr", dto2Primitive(version, DatasetFieldConstant.depositor)); + writeFullElement(xmlw, "depDate", dto2Primitive(version, DatasetFieldConstant.dateOfDeposit)); + xmlw.writeEndElement(); // diststmt + writeSeriesElement(xmlw, version); + xmlw.writeEndElement(); // citation //End Citation Block @@ -201,9 +211,16 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeSubjectElement(xmlw, version); //Subject and Keywords writeAbstractElement(xmlw, version); // Description - writeFullElement(xmlw, "notes", dto2Primitive(version, DatasetFieldConstant.notesText)); - writeSummaryDescriptionElement(xmlw, version); + writeFullElement(xmlw, "notes", dto2Primitive(version, DatasetFieldConstant.notesText)); + //////// + xmlw.writeEndElement(); // stdyInfo + + writeMethodElement(xmlw, version); + writeDataAccess(xmlw , version); + writeOtherStudyMaterial(xmlw , version); + + /* writeRelPublElement(xmlw, version); writeOtherIdElement(xmlw, version); @@ -244,11 +261,53 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeFullElement(xmlw, "contact", version.getContactForAccess()); writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); writeFullElement(xmlw, "complete", version.getStudyCompletion()); - + */ xmlw.writeEndElement(); // stdyDscr } + + private static void writeOtherStudyMaterial(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException { + xmlw.writeStartElement("othrStdyMat"); + writeFullElementList(xmlw, "relMat", dto2PrimitiveList(version, DatasetFieldConstant.relatedMaterial)); + writeFullElementList(xmlw, "relStdy", dto2PrimitiveList(version, DatasetFieldConstant.relatedDatasets)); + writeRelPublElement(xmlw, version); + writeFullElementList(xmlw, "othRefs", dto2PrimitiveList(version, DatasetFieldConstant.otherReferences)); + xmlw.writeEndElement(); //othrStdyMat + } + + private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO version) throws XMLStreamException { + xmlw.writeStartElement("dataAccs"); + if (version.getTermsOfUse() != null && !version.getTermsOfUse().trim().equals("")) { + xmlw.writeStartElement("notes"); + writeAttribute(xmlw, "type", NOTE_TYPE_TERMS_OF_USE); + writeAttribute(xmlw, "level", LEVEL_DV); + xmlw.writeCharacters(version.getTermsOfUse()); + xmlw.writeEndElement(); //notes + } + /*else if (xmlr.getLocalName().equals("notes")) { + String noteType = xmlr.getAttributeValue(null, "type"); + if (NOTE_TYPE_TERMS_OF_USE.equalsIgnoreCase(noteType) ) { + if ( LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) { + dvDTO.setTermsOfUse(parseText(xmlr, "notes")); + }*/ + writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); + writeFullElement(xmlw, "origArch", version.getOriginalArchive()); + writeFullElement(xmlw, "avlStatus", version.getAvailabilityStatus()); + writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); + writeFullElement(xmlw, "complete", version.getStudyCompletion()); + xmlw.writeStartElement("useStmt"); + writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); + writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); + writeFullElement(xmlw, "restrctn", version.getRestrictions()); + writeFullElement(xmlw, "contact", version.getContactForAccess()); + writeFullElement(xmlw, "citeReq", version.getCitationRequirements()); + writeFullElement(xmlw, "deposReq", version.getDepositorRequirements()); + writeFullElement(xmlw, "conditions", version.getConditions()); + writeFullElement(xmlw, "disclaimer", version.getDisclaimer()); + xmlw.writeEndElement(); //useStmt + xmlw.writeEndElement(); //dataAccs + } private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datasetDto) throws XMLStreamException { DatasetVersionDTO version = datasetDto.getDatasetVersion(); @@ -443,9 +502,19 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO writeFullElement(xmlw, "dataCollector", dto2Primitive(version, DatasetFieldConstant.dataCollector)); writeFullElement(xmlw, "collectorTraining", dto2Primitive(version, DatasetFieldConstant.collectorTraining)); writeFullElement(xmlw, "frequenc", dto2Primitive(version, DatasetFieldConstant.frequencyOfDataCollection)); - writeFullElement(xmlw, "sampProc", dto2Primitive(version, DatasetFieldConstant.samplingProcedure)); + writeFullElement(xmlw, "sampProc", dto2Primitive(version, DatasetFieldConstant.samplingProcedure)); + writeTargetSampleElement(xmlw, version); - writeFullElement(xmlw, "deviat", dto2Primitive(version, DatasetFieldConstant.deviationsFromSampleDesign)); + + writeFullElement(xmlw, "deviat", dto2Primitive(version, DatasetFieldConstant.deviationsFromSampleDesign)); + + xmlw.writeStartElement("sources"); + writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources)); + writeFullElement(xmlw, "srcOrig", dto2Primitive(version, DatasetFieldConstant.originOfSources)); + writeFullElement(xmlw, "srcChar", dto2Primitive(version, DatasetFieldConstant.characteristicOfSources)); + writeFullElement(xmlw, "srcDocu", dto2Primitive(version, DatasetFieldConstant.accessToSources)); + xmlw.writeEndElement(); //sources + writeFullElement(xmlw, "collMode", dto2Primitive(version, DatasetFieldConstant.collectionMode)); writeFullElement(xmlw, "resInstru", dto2Primitive(version, DatasetFieldConstant.researchInstrument)); writeFullElement(xmlw, "collSitu", dto2Primitive(version, DatasetFieldConstant.dataCollectionSituation)); @@ -456,7 +525,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO xmlw.writeEndElement(); //dataColl xmlw.writeStartElement("anlyInfo"); - writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); + //writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); writeFullElement(xmlw, "respRate", dto2Primitive(version, DatasetFieldConstant.responseRate)); writeFullElement(xmlw, "estSmpErr", dto2Primitive(version, DatasetFieldConstant.samplingErrorEstimates)); writeFullElement(xmlw, "dataAppr", dto2Primitive(version, DatasetFieldConstant.otherDataAppraisal)); @@ -691,6 +760,7 @@ private static void writeProducersElement(XMLStreamWriter xmlw, DatasetVersionDT } writeFullElement(xmlw, "prodDate", dto2Primitive(version, DatasetFieldConstant.productionDate)); writeFullElement(xmlw, "prodPlac", dto2Primitive(version, DatasetFieldConstant.productionPlace)); + writeSoftwareElement(xmlw, version); writeGrantElement(xmlw, version); xmlw.writeEndElement(); //prodStmt @@ -703,7 +773,7 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio if ("citation".equals(key)) { for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.distributor.equals(fieldDTO.getTypeName())) { - xmlw.writeStartElement("distrbtr"); + //xmlw.writeStartElement("distrbtr"); for (HashSet foo : fieldDTO.getMultipleCompound()) { String distributorName = ""; String distributorAffiliation = ""; @@ -746,8 +816,9 @@ private static void writeDistributorsElement(XMLStreamWriter xmlw, DatasetVersio xmlw.writeEndElement(); //AuthEnty } } - xmlw.writeEndElement(); //rspStmt + //xmlw.writeEndElement(); //rspStmt } + } } } @@ -987,6 +1058,7 @@ private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersio if ("socialscience".equals(key)) { for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.targetSampleSize.equals(fieldDTO.getTypeName())) { + xmlw.writeStartElement("targetSampleSize"); String sizeFormula = ""; String actualSize = ""; Set foo = fieldDTO.getSingleCompound(); @@ -999,6 +1071,7 @@ private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersio actualSize = next.getSinglePrimitive(); } } + if (!sizeFormula.isEmpty()) { xmlw.writeStartElement("sampleSizeFormula"); xmlw.writeCharacters(sizeFormula); @@ -1009,6 +1082,7 @@ private static void writeTargetSampleElement(XMLStreamWriter xmlw, DatasetVersio xmlw.writeCharacters(actualSize); xmlw.writeEndElement(); //sampleSize } + xmlw.writeEndElement(); // targetSampleSize } } } diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml index 3023d9eebe1..05e32869bb7 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml @@ -25,7 +25,11 @@ Johnny Hawk - + + Odin Raven + Jimmy Finch + Added, Depositor + @@ -50,15 +54,16 @@ 59.8 43.8 - - Odin Raven - - Jimmy Finch - Added, Depositor - + + + + + + + From a53c85720514ace75a263626c4b476fa58204a35 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Tue, 18 Feb 2020 15:56:12 -0500 Subject: [PATCH 092/157] import export --- .../iq/dataverse/DatasetFieldConstant.java | 2 +- .../api/imports/ImportDDIServiceBean.java | 80 ++++++++++++++----- .../dataverse/export/ddi/DdiExportUtil.java | 40 +++++++--- .../export/ddi/DdiExportUtilTest.java | 12 +++ .../dataverse/export/ddi/dataset-finch1.xml | 18 +++-- 5 files changed, 114 insertions(+), 38 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java index 69de63c5fa6..6d26c0cba58 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetFieldConstant.java @@ -97,7 +97,7 @@ public class DatasetFieldConstant implements java.io.Serializable { public final static String topicClassVocab="topicClassVocab"; public final static String topicClassVocabURI="topicClassVocabURI"; public final static String descriptionText="dsDescriptionValue"; - public final static String descriptionDate="descriptionDate"; + public final static String descriptionDate="dsDescriptionDate"; public final static String timePeriodCovered="timePeriodCovered"; // SEK added 6/13/2016 public final static String timePeriodCoveredStart="timePeriodCoveredStart"; public final static String timePeriodCoveredEnd="timePeriodCoveredEnd"; diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 178a23b6f2f..3617e561837 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -418,10 +418,10 @@ else if (xmlr.getLocalName().equals("relStdy")) { publications.add(set); - } else if (xmlr.getLocalName().equals("otherRefs")) { + } else if (xmlr.getLocalName().equals("othRefs")) { List otherRefs = new ArrayList<>(); - otherRefs.add(parseText(xmlr, "otherRefs")); + otherRefs.add(parseText(xmlr, "othRefs")); getCitation(dvDTO).addField(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.otherReferences, otherRefs)); } @@ -620,6 +620,42 @@ private void processNotes (XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws this.addNote(formattedNotes, dvDTO); } } + + private void processNotesSocialScience(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { + //String formattedNotes = this.formatNotesfromXML(xmlr); + if (xmlr==null){ + throw new NullPointerException("XMLStreamReader xmlr cannot be null"); + } + FieldDTO notesSubject = null; + String attrVal; + + // Check for "subject" + attrVal = xmlr.getAttributeValue(null, "subject"); + if (attrVal != null){ + notesSubject = FieldDTO.createPrimitiveFieldDTO("socialScienceNotesSubject", attrVal); + } + + FieldDTO notesType = null; + // Check for "type" + attrVal = xmlr.getAttributeValue(null, "type"); + if (attrVal != null){ + notesType = FieldDTO.createPrimitiveFieldDTO("socialScienceNotesType", attrVal); + } + + FieldDTO notesText = null; + // Add notes, if they exist + attrVal = parseText(xmlr, "notes"); + if ((attrVal != null) && (!attrVal.isEmpty())){ + notesText = FieldDTO.createPrimitiveFieldDTO("socialScienceNotesText", attrVal); + } + + if (notesSubject != null || notesType != null || notesText != null ){ + + //this.addNoteSocialScience(formattedNotes, dvDTO); + + getSocialScience(dvDTO).addField(FieldDTO.createCompoundFieldDTO("socialScienceNotes", notesSubject, notesType, notesText )); + } + } private void addNote(String noteText, DatasetVersionDTO dvDTO ) { MetadataBlockDTO citation = getCitation(dvDTO); @@ -640,6 +676,8 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw List kindOfData = new ArrayList<>(); List> geoBoundBox = new ArrayList<>(); List> geoCoverages = new ArrayList<>(); + List timePeriod = new ArrayList<>(); + List dateOfCollection = new ArrayList<>(); FieldDTO timePeriodStart = null; FieldDTO timePeriodEnd = null; FieldDTO dateOfCollectionStart = null; @@ -654,6 +692,8 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw timePeriodStart = FieldDTO.createPrimitiveFieldDTO("timePeriodCoveredStart", parseDate(xmlr, "timePrd")); } else if (EVENT_END.equals(eventAttr)) { timePeriodEnd = FieldDTO.createPrimitiveFieldDTO("timePeriodCoveredEnd", parseDate(xmlr, "timePrd")); + timePeriod.add(FieldDTO.createMultipleCompoundFieldDTO("timePeriodCovered", timePeriodStart, timePeriodEnd)); + } } else if (xmlr.getLocalName().equals("collDate")) { String eventAttr = xmlr.getAttributeValue(null, "event"); @@ -661,6 +701,7 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw dateOfCollectionStart = FieldDTO.createPrimitiveFieldDTO("dateOfCollectionStart", parseDate(xmlr, "collDate")); } else if (EVENT_END.equals(eventAttr)) { dateOfCollectionEnd = FieldDTO.createPrimitiveFieldDTO("dateOfCollectionEnd", parseDate(xmlr, "collDate")); + dateOfCollection.add(FieldDTO.createMultipleCompoundFieldDTO("dateOfCollection", dateOfCollectionStart, dateOfCollectionEnd )); } } else if (xmlr.getLocalName().equals("nation")) { @@ -684,11 +725,11 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("sumDscr")) { - if (timePeriodStart!=null || timePeriodEnd!=null) { - getCitation(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO("timePeriodCovered", timePeriodStart, timePeriodEnd)); + for (FieldDTO time : timePeriod) { + getCitation(dvDTO).addField( time); } - if (dateOfCollectionStart!=null || dateOfCollectionEnd!=null) { - getCitation(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO("dateOfCollection", dateOfCollectionStart, dateOfCollectionEnd)); + for (FieldDTO date : dateOfCollection) { + getCitation(dvDTO).addField(date); } if (geoUnit.size() > 0) { @@ -748,7 +789,7 @@ private void processMethod(XMLStreamReader xmlr, DatasetVersionDTO dvDTO ) throw if (NOTE_TYPE_EXTENDED_METADATA.equalsIgnoreCase(noteType) ) { processCustomField(xmlr, dvDTO); } else { - processNotes(xmlr, dvDTO); + processNotesSocialScience(xmlr, dvDTO); // addNote("Subject: Study Level Error Note, Notes: "+ parseText( xmlr,"notes" ) +";", dvDTO); } @@ -959,8 +1000,8 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("dataCollectionSituation", parseText( xmlr, "collSitu" ))); } else if (xmlr.getLocalName().equals("actMin")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("actionsToMinimizeLoss", parseText( xmlr, "actMin" ))); - } else if (xmlr.getLocalName().equals("ConOps")) { - socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("controlOperations", parseText( xmlr, "ConOps" ))); + } else if (xmlr.getLocalName().equals("conOps")) { + socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("controlOperations", parseText( xmlr, "conOps" ))); } else if (xmlr.getLocalName().equals("weight")) { String thisValue = parseText( xmlr, "weight" ); if (!StringUtil.isEmpty(thisValue)) { @@ -972,6 +1013,8 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro //socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("weighting", parseText( xmlr, "weight" ))); } else if (xmlr.getLocalName().equals("cleanOps")) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("cleaningOperations", parseText( xmlr, "cleanOps" ))); + } else if (xmlr.getLocalName().equals("collectorTraining")) { + socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("collectorTraining", parseText( xmlr, "collectorTraining" ))); } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("dataColl")) { @@ -987,6 +1030,7 @@ private void processDataColl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro if (!StringUtil.isEmpty(dataCollector)) { socialScience.getFields().add(FieldDTO.createPrimitiveFieldDTO("dataCollector", dataCollector)); } + return; } } @@ -1192,11 +1236,9 @@ private void processDistStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th HashSet set = new HashSet<>(); addToSet(set, "distributorAbbreviation", xmlr.getAttributeValue(null, "abbr")); addToSet(set, "distributorAffiliation", xmlr.getAttributeValue(null, "affiliation")); - - Map distDetails = parseCompoundText(xmlr, "distrbtr"); - addToSet(set, "distributorName", distDetails.get("name")); - addToSet(set, "distributorURL", distDetails.get("url")); - addToSet(set, "distributorLogoURL", distDetails.get("logo")); + addToSet(set, "distributorURL", xmlr.getAttributeValue(null, "URI")); + addToSet(set, "distributorLogoURL", xmlr.getAttributeValue(null, "role")); + addToSet(set, "distributorName", xmlr.getElementText()); distributors.add(set); } else if (xmlr.getLocalName().equals("contact")) { @@ -1240,11 +1282,9 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th HashSet set = new HashSet<>(); addToSet(set,"producerAbbreviation", xmlr.getAttributeValue(null, "abbr")); addToSet(set,"producerAffiliation", xmlr.getAttributeValue(null, "affiliation")); - - Map prodDetails = parseCompoundText(xmlr, "producer"); - addToSet(set,"producerName", prodDetails.get("name")); - addToSet(set,"producerURL", prodDetails.get("url" )); - addToSet(set,"producerLogoURL", prodDetails.get("logo")); + addToSet(set,"producerLogoURL", xmlr.getAttributeValue(null, "role")); + addToSet(set,"producerURL", xmlr.getAttributeValue(null, "URI")); + addToSet(set,"producerName", xmlr.getElementText()); if (!set.isEmpty()) producers.add(set); } else if (xmlr.getLocalName().equals("prodDate")) { @@ -1254,7 +1294,7 @@ private void processProdStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th } else if (xmlr.getLocalName().equals("software")) { HashSet set = new HashSet<>(); addToSet(set,"softwareVersion", xmlr.getAttributeValue(null, "version")); - addToSet(set,"softwareName", xmlr.getAttributeValue(null, "version")); + addToSet(set,"softwareName", parseText(xmlr)); if (!set.isEmpty()) { software.add(set); } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index d35b3702d29..8bdde3094bb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -194,7 +194,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) xmlw.writeStartElement("distStmt"); writeDistributorsElement(xmlw, version); writeContactsElement(xmlw, version); - writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); + writeFullElement(xmlw, "distDate", dto2Primitive(version, DatasetFieldConstant.distributionDate)); writeFullElement(xmlw, "depositr", dto2Primitive(version, DatasetFieldConstant.depositor)); writeFullElement(xmlw, "depDate", dto2Primitive(version, DatasetFieldConstant.dateOfDeposit)); @@ -422,26 +422,46 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset if("geospatial".equals(key)){ for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.geographicCoverage.equals(fieldDTO.getTypeName())) { + for (HashSet foo : fieldDTO.getMultipleCompound()) { + HashMap geoMap = new HashMap<>(); for (Iterator iterator = foo.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); if (DatasetFieldConstant.country.equals(next.getTypeName())) { - writeFullElement(xmlw, "nation", next.getSinglePrimitive()); + geoMap.put("country", next.getSinglePrimitive()); + //writeFullElement(xmlw, "nation", next.getSinglePrimitive()); } if (DatasetFieldConstant.city.equals(next.getTypeName())) { - writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); + geoMap.put("city", next.getSinglePrimitive()); + //writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); } if (DatasetFieldConstant.state.equals(next.getTypeName())) { - writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); + geoMap.put("state", next.getSinglePrimitive()); + //writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); } if (DatasetFieldConstant.otherGeographicCoverage.equals(next.getTypeName())) { - writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); + geoMap.put("otherGeographicCoverage", next.getSinglePrimitive()); + //writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); } } + if (geoMap.get("country") != null) { + writeFullElement(xmlw, "nation", geoMap.get("country")); + } + if (geoMap.get("city") != null) { + writeFullElement(xmlw, "geogCover", geoMap.get("city")); + } + if (geoMap.get("state") != null) { + writeFullElement(xmlw, "geogCover", geoMap.get("state")); + } + if (geoMap.get("otherGeographicCoverage") != null) { + writeFullElement(xmlw, "geogCover", geoMap.get("otherGeographicCoverage")); + } } } if (DatasetFieldConstant.geographicBoundingBox.equals(fieldDTO.getTypeName())) { + for (HashSet foo : fieldDTO.getMultipleCompound()) { + xmlw.writeStartElement("geoBndBox"); for (Iterator iterator = foo.iterator(); iterator.hasNext();) { FieldDTO next = iterator.next(); if (DatasetFieldConstant.westLongitude.equals(next.getTypeName())) { @@ -458,7 +478,9 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset } } + xmlw.writeEndElement(); } + } } writeFullElementList(xmlw, "geogUnit", dto2PrimitiveList(datasetVersionDTO, DatasetFieldConstant.geographicUnit)); @@ -519,7 +541,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO writeFullElement(xmlw, "resInstru", dto2Primitive(version, DatasetFieldConstant.researchInstrument)); writeFullElement(xmlw, "collSitu", dto2Primitive(version, DatasetFieldConstant.dataCollectionSituation)); writeFullElement(xmlw, "actMin", dto2Primitive(version, DatasetFieldConstant.actionsToMinimizeLoss)); - writeFullElement(xmlw, "conOps", dto2Primitive(version, DatasetFieldConstant.controlOperations)); + writeFullElement(xmlw, "conOps", dto2Primitive(version, DatasetFieldConstant.controlOperations)); writeFullElement(xmlw, "weight", dto2Primitive(version, DatasetFieldConstant.weighting)); writeFullElement(xmlw, "cleanOps", dto2Primitive(version, DatasetFieldConstant.cleaningOperations)); @@ -527,7 +549,7 @@ private static void writeMethodElement(XMLStreamWriter xmlw , DatasetVersionDTO xmlw.writeStartElement("anlyInfo"); //writeFullElement(xmlw, "anylInfo", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); writeFullElement(xmlw, "respRate", dto2Primitive(version, DatasetFieldConstant.responseRate)); - writeFullElement(xmlw, "estSmpErr", dto2Primitive(version, DatasetFieldConstant.samplingErrorEstimates)); + writeFullElement(xmlw, "EstSmpErr", dto2Primitive(version, DatasetFieldConstant.samplingErrorEstimates)); writeFullElement(xmlw, "dataAppr", dto2Primitive(version, DatasetFieldConstant.otherDataAppraisal)); xmlw.writeEndElement(); //anlyInfo writeNotesElement(xmlw, version); @@ -576,7 +598,7 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO writeAttribute(xmlw,"vocab",keywordVocab); } if(!keywordURI.isEmpty()){ - writeAttribute(xmlw,"URI",keywordURI); + writeAttribute(xmlw,"vocabURI",keywordURI); } xmlw.writeCharacters(keywordValue); xmlw.writeEndElement(); //Keyword @@ -607,7 +629,7 @@ private static void writeSubjectElement(XMLStreamWriter xmlw, DatasetVersionDTO writeAttribute(xmlw,"vocab",topicClassificationVocab); } if(!topicClassificationURI.isEmpty()){ - writeAttribute(xmlw,"URI",topicClassificationURI); + writeAttribute(xmlw,"vocabURI",topicClassificationURI); } xmlw.writeCharacters(topicClassificationValue); xmlw.writeEndElement(); //topcClas diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java index b3155e6d7ae..09be153b647 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java @@ -35,6 +35,18 @@ public void testJson2DdiNoFiles() throws Exception { assertEquals(datasetAsDdi, result); } + /* @Test + public void testJson2DdiNoFiles2() throws Exception { + File datasetVersionJson = new File("dataset.json"); + String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); + File ddiFile = new File("exportfull.xml"); + String datasetAsDdi = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(ddiFile.getAbsolutePath())))); + logger.info(datasetAsDdi); + String result = DdiExportUtil.datasetDtoAsJson2ddi(datasetVersionAsJson); + logger.info(result); + assertEquals(datasetAsDdi, result); + }*/ + @Test public void testJson2ddiHasFiles() throws Exception { /** diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml index 05e32869bb7..cc09b8b2328 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml @@ -34,9 +34,9 @@ Medicine, Health and Life Sciences - Keyword Value 1 - Keyword Value Two - TC Value 1 + Keyword Value 1 + Keyword Value Two + TC Value 1 Darwin's finches (also known as the Galápagos finches) are a group of about fifteen species of passerine birds. @@ -45,14 +45,16 @@ 20070831 20130630 Kind of Data - Cambridge USA + Cambridge MA Other Geographic Coverage - 41.6 - 60.3 - 59.8 - 43.8 + + 41.6 + 60.3 + 59.8 + 43.8 + From 4b0528b6566f03def8634ae478a41cbc872465c6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 20 Feb 2020 15:23:22 -0500 Subject: [PATCH 093/157] fix merge issue --- doc/release-notes/6485-multiple-stores.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/doc/release-notes/6485-multiple-stores.md b/doc/release-notes/6485-multiple-stores.md index ea2d224d612..e9c7e654d96 100644 --- a/doc/release-notes/6485-multiple-stores.md +++ b/doc/release-notes/6485-multiple-stores.md @@ -29,8 +29,4 @@ Any additional S3 options you have set will need to be replaced as well, followi Once these options are set, restarting the glassfish service is all that is needed to complete the change. -<<<<<<< HEAD Note that the "\-Ddataverse.files.directory", if defined, continues to control where temporary files are stored (in the /temp subdir of that directory), independent of the location of any 'file' store defined above. -======= -Note that the "\-Ddataverse.files.directory", if defined, continues to control where temporary files are stored (in the /temp subdir of that directory), independent of the location of any 'file' store defined above. ->>>>>>> branch 'IQSS/6485' of https://github.com/TexasDigitalLibrary/dataverse.git From 019862b11e4239c1c75b03d31aa080a8fda71392 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 20 Feb 2020 15:23:33 -0500 Subject: [PATCH 094/157] direct upload release notes --- doc/release-notes/6489-release-notes.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/release-notes/6489-release-notes.md diff --git a/doc/release-notes/6489-release-notes.md b/doc/release-notes/6489-release-notes.md new file mode 100644 index 00000000000..3c21cd52bdb --- /dev/null +++ b/doc/release-notes/6489-release-notes.md @@ -0,0 +1,17 @@ +# S3 Direct Upload support + +S3 stores can now optionally be configured to support direct upload of files, as one option for supporting upload of larger files. + +General information about this capability can be found in the Big Data Support Guide with specific information about how to enable it in the Configuration Guide - File Storage section. + +**Upgrade Information:** + +Direct upload to S3 is enabled per store by one new jvm option: + + ./asadmin create-jvm-options "\-Ddataverse.files..upload-redirect=true" + +The existing :MaxFileUploadSizeInBytes property and dataverse.files..url-expiration-minutes jvm option for the same store also apply to direct upload. + +Direct upload via the Dataverse web interface is transparent to the user and handled automatically by the browser. Some minor differences in file upload exist: directly uploaded files are not unzipped and Dataverse does not scan their content to help in assigning a MIME type. Ingest of tabular files and metadata extraction from FITS files will occur, but can be turned off for files above a specified size limit through the new dataverse.files..ingestsizelimit jvm option. + +API calls to support direct upload also exist, and, if direct upload is enabled for a store in Dataverse, the latest DVUploader (v1.0.8) provides a'-directupload' flag that enables its use. \ No newline at end of file From 9b98584c16ce3e0c2212ce0e274e262dfce9b880 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Fri, 21 Feb 2020 14:18:04 -0500 Subject: [PATCH 095/157] export import ddi --- .../api/imports/ImportDDIServiceBean.java | 58 +- .../dataverse/export/ddi/DdiExportUtil.java | 49 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 10 +- .../export/ddi/DdiExportUtilTest.java | 10 +- .../dataset-create-new-all-ddi-fields.json | 983 ++++++++++++++++++ .../iq/dataverse/export/ddi/exportfull.xml | 156 +++ 6 files changed, 1199 insertions(+), 67 deletions(-) create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json create mode 100644 src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 3617e561837..528070c31d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -27,6 +27,8 @@ import javax.xml.stream.XMLStreamException; import javax.xml.stream.XMLStreamReader; import javax.xml.stream.XMLInputFactory; + +import edu.harvard.iq.dataverse.util.json.ControlledVocabularyException; import org.apache.commons.lang.StringUtils; /** @@ -344,7 +346,8 @@ private void processStdyDscr(ImportType importType, XMLStreamReader xmlr, Datase else if (xmlr.getLocalName().equals("stdyInfo")) processStdyInfo(xmlr, datasetDTO.getDatasetVersion()); else if (xmlr.getLocalName().equals("method")) processMethod(xmlr, datasetDTO.getDatasetVersion()); - else if (xmlr.getLocalName().equals("dataAccs")) processDataAccs(xmlr, datasetDTO.getDatasetVersion()); + else if (xmlr.getLocalName().equals("dataAccs")) processDataAccs(xmlr, datasetDTO.getDatasetVersion()); + else if (xmlr.getLocalName().equals("notes")) processStdyNotes(xmlr, datasetDTO.getDatasetVersion()); else if (xmlr.getLocalName().equals("othrStdyMat")) processOthrStdyMat(xmlr, datasetDTO.getDatasetVersion()); else if (xmlr.getLocalName().equals("notes")) processNotes(xmlr, datasetDTO.getDatasetVersion()); @@ -620,6 +623,16 @@ private void processNotes (XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws this.addNote(formattedNotes, dvDTO); } } + private void processStdyNotes(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { + FieldDTO notesText = null; + // Add notes, if they exist + String attrVal = parseText(xmlr, "notes"); + if ((attrVal != null) && (!attrVal.isEmpty())){ + notesText = FieldDTO.createPrimitiveFieldDTO("datasetLevelErrorNotes", attrVal); + getSocialScience(dvDTO).addField(notesText); + } + } + private void processNotesSocialScience(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throws XMLStreamException { //String formattedNotes = this.formatNotesfromXML(xmlr); @@ -683,7 +696,10 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw FieldDTO dateOfCollectionStart = null; FieldDTO dateOfCollectionEnd = null; + HashSet geoCoverageSet = null; + String otherGeographicCoverage = null; for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { + if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("timePrd")) { @@ -705,13 +721,28 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw } } else if (xmlr.getLocalName().equals("nation")) { - HashSet set = new HashSet<>(); - set.add(FieldDTO.createVocabFieldDTO("country", parseText(xmlr))); - geoCoverages.add(set); + if (otherGeographicCoverage != null && !otherGeographicCoverage.equals("")) { + geoCoverageSet.add(FieldDTO.createPrimitiveFieldDTO("otherGeographicCoverage", otherGeographicCoverage)); + otherGeographicCoverage = null; + } + if (geoCoverageSet != null && geoCoverageSet.size() > 0) { + geoCoverages.add(geoCoverageSet); + } + geoCoverageSet = new HashSet<>(); + //HashSet set = new HashSet<>(); + //set.add(FieldDTO.createVocabFieldDTO("country", parseText(xmlr))); + geoCoverageSet.add(FieldDTO.createVocabFieldDTO("country", parseText(xmlr))); + } else if (xmlr.getLocalName().equals("geogCover")) { - HashSet set = new HashSet<>(); - set.add(FieldDTO.createPrimitiveFieldDTO("otherGeographicCoverage", parseText(xmlr))); - geoCoverages.add(set); + if (geoCoverageSet == null) { + geoCoverageSet = new HashSet<>(); + } + if (otherGeographicCoverage != null) { + otherGeographicCoverage = otherGeographicCoverage + "; " + parseText(xmlr); + } else { + otherGeographicCoverage = parseText(xmlr); + } + } else if (xmlr.getLocalName().equals("geogUnit")) { geoUnit.add(parseText(xmlr)); } else if (xmlr.getLocalName().equals("geoBndBox")) { @@ -744,8 +775,15 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw if (kindOfData.size() > 0) { getCitation(dvDTO).addField(FieldDTO.createMultiplePrimitiveFieldDTO("kindOfData", kindOfData)); } - if (geoCoverages.size()>0) { - getGeospatial(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO("geographicCoverage", geoCoverages)); + if (otherGeographicCoverage != null && !otherGeographicCoverage.equals("") ) { + geoCoverageSet.add(FieldDTO.createPrimitiveFieldDTO("otherGeographicCoverage", otherGeographicCoverage)); + } + if (geoCoverageSet != null && geoCoverageSet.size() > 0) { + //FieldDTO geoCoverageDTO = FieldDTO.createMultipleCompoundFieldDTO(DatasetFieldConstant.geographicCoverage, geoCoverageList); + geoCoverages.add(geoCoverageSet); + } + if (geoCoverages.size() > 0) { + getGeospatial(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO(DatasetFieldConstant.geographicCoverage, geoCoverages)); } if (geoBoundBox.size()>0) { getGeospatial(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO("geographicBoundingBox", geoBoundBox)); @@ -756,8 +794,6 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw } } - - private HashSet processGeoBndBox(XMLStreamReader xmlr) throws XMLStreamException { HashSet set = new HashSet<>(); diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index fe61f201466..db019d5a39a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -221,48 +221,7 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeDataAccess(xmlw , version); writeOtherStudyMaterial(xmlw , version); - /* - writeRelPublElement(xmlw, version); - - writeOtherIdElement(xmlw, version); - writeDistributorsElement(xmlw, version); - writeContactsElement(xmlw, version); - writeFullElement(xmlw, "depositr", dto2Primitive(version, DatasetFieldConstant.depositor)); - writeFullElement(xmlw, "depDate", dto2Primitive(version, DatasetFieldConstant.dateOfDeposit)); - - writeFullElementList(xmlw, "relMat", dto2PrimitiveList(version, DatasetFieldConstant.relatedMaterial)); - writeFullElementList(xmlw, "relStdy", dto2PrimitiveList(version, DatasetFieldConstant.relatedDatasets)); - writeFullElementList(xmlw, "othRefs", dto2PrimitiveList(version, DatasetFieldConstant.otherReferences)); - writeSeriesElement(xmlw, version); - writeSoftwareElement(xmlw, version); - writeFullElementList(xmlw, "dataSrc", dto2PrimitiveList(version, DatasetFieldConstant.dataSources)); - writeFullElement(xmlw, "srcOrig", dto2Primitive(version, DatasetFieldConstant.originOfSources)); - writeFullElement(xmlw, "srcChar", dto2Primitive(version, DatasetFieldConstant.characteristicOfSources)); - writeFullElement(xmlw, "srcDocu", dto2Primitive(version, DatasetFieldConstant.accessToSources)); - xmlw.writeEndElement(); // stdyInfo - // End Info Block - - //Social Science Metadata block - - writeMethodElement(xmlw, version); - - //Terms of Use and Access - writeFullElement(xmlw, "useStmt", version.getTermsOfUse()); - writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); - writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); - writeFullElement(xmlw, "restrctn", version.getRestrictions()); - writeFullElement(xmlw, "citeReq", version.getCitationRequirements()); - writeFullElement(xmlw, "deposReq", version.getDepositorRequirements()); - writeFullElement(xmlw, "dataAccs", version.getTermsOfAccess()); - writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); - writeFullElement(xmlw, "conditions", version.getConditions()); - writeFullElement(xmlw, "disclaimer", version.getDisclaimer()); - writeFullElement(xmlw, "origArch", version.getOriginalArchive()); - writeFullElement(xmlw, "avlStatus", version.getAvailabilityStatus()); - writeFullElement(xmlw, "contact", version.getContactForAccess()); - writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); - writeFullElement(xmlw, "complete", version.getStudyCompletion()); - */ + writeFullElement(xmlw, "notes", dto2Primitive(version, DatasetFieldConstant.datasetLevelErrorNotes)); xmlw.writeEndElement(); // stdyDscr @@ -430,21 +389,18 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset FieldDTO next = iterator.next(); if (DatasetFieldConstant.country.equals(next.getTypeName())) { geoMap.put("country", next.getSinglePrimitive()); - //writeFullElement(xmlw, "nation", next.getSinglePrimitive()); } if (DatasetFieldConstant.city.equals(next.getTypeName())) { geoMap.put("city", next.getSinglePrimitive()); - //writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); } if (DatasetFieldConstant.state.equals(next.getTypeName())) { geoMap.put("state", next.getSinglePrimitive()); - //writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); } if (DatasetFieldConstant.otherGeographicCoverage.equals(next.getTypeName())) { geoMap.put("otherGeographicCoverage", next.getSinglePrimitive()); - //writeFullElement(xmlw, "geogCover", next.getSinglePrimitive()); } } + if (geoMap.get("country") != null) { writeFullElement(xmlw, "nation", geoMap.get("country")); } @@ -457,6 +413,7 @@ private static void writeSummaryDescriptionElement(XMLStreamWriter xmlw, Dataset if (geoMap.get("otherGeographicCoverage") != null) { writeFullElement(xmlw, "geogCover", geoMap.get("otherGeographicCoverage")); } + } } if (DatasetFieldConstant.geographicBoundingBox.equals(fieldDTO.getTypeName())) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 51d3a214d5d..713fb770cb0 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -446,9 +446,9 @@ public void testCreatePublishDestroyDataset() { */ boolean nameRequiredForContactToAppear = true; if (nameRequiredForContactToAppear) { - assertEquals("Finch, Fiona", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.stdyInfo.contact")); + assertEquals("Finch, Fiona", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.citation.distStmt.contact")); } else { - assertEquals("finch@mailinator.com", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.stdyInfo.contact.@email")); + assertEquals("finch@mailinator.com", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.citation.distStmt.contact.@email")); } assertEquals(datasetPersistentId, XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.docDscr.citation.titlStmt.IDNo")); @@ -659,10 +659,10 @@ public void testExcludeEmail() { exportDatasetAsDdi.then().assertThat() .statusCode(OK.getStatusCode()); - assertEquals("Dataverse, Admin", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.stdyInfo.contact")); + assertEquals("Dataverse, Admin", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.citation.distStmt.contact")); // no "sammi@sample.com" to be found https://github.com/IQSS/dataverse/issues/3443 - assertEquals("[]", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.stdyInfo.contact.@email")); - assertEquals("Sample Datasets, inc.", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.stdyInfo.contact.@affiliation")); + assertEquals("[]", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.citation.distStmt.contact.@email")); + assertEquals("Sample Datasets, inc.", XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.stdyDscr.citation.distStmt.contact.@affiliation")); assertEquals(datasetPersistentId, XmlPath.from(exportDatasetAsDdi.body().asString()).getString("codeBook.docDscr.citation.titlStmt.IDNo")); List datasetContactsFromNativeGet = with(getDatasetJsonAfterPublishing.body().asString()).param("datasetContact", "datasetContact") diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java index 09be153b647..a76ce8475f2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtilTest.java @@ -35,17 +35,17 @@ public void testJson2DdiNoFiles() throws Exception { assertEquals(datasetAsDdi, result); } - /* @Test - public void testJson2DdiNoFiles2() throws Exception { - File datasetVersionJson = new File("dataset.json"); + @Test + public void testExportDDI() throws Exception { + File datasetVersionJson = new File("src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json"); String datasetVersionAsJson = new String(Files.readAllBytes(Paths.get(datasetVersionJson.getAbsolutePath()))); - File ddiFile = new File("exportfull.xml"); + File ddiFile = new File("src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml"); String datasetAsDdi = XmlPrinter.prettyPrintXml(new String(Files.readAllBytes(Paths.get(ddiFile.getAbsolutePath())))); logger.info(datasetAsDdi); String result = DdiExportUtil.datasetDtoAsJson2ddi(datasetVersionAsJson); logger.info(result); assertEquals(datasetAsDdi, result); - }*/ + } @Test public void testJson2ddiHasFiles() throws Exception { diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json new file mode 100644 index 00000000000..31d43eb5377 --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json @@ -0,0 +1,983 @@ +{ + "id": 11, + "identifier": "WKUKGV", + "persistentUrl": "https://doi.org/10.5072/FK2/WKUKGV", + "protocol": "doi", + "authority": "10.5072/FK2", + "publisher": "Root", + "publicationDate": "2020-02-19", + "datasetVersion": { + "id": 2, + "versionNumber": 1, + "versionMinorNumber": 0, + "versionState": "RELEASED", + "productionDate": "Production Date", + "lastUpdateTime": "2015-09-24T17:07:57Z", + "releaseTime": "2020-02-19", + "createTime": "2015-09-24T16:47:51Z", + "license": "CC0", + "termsOfUse": "CC0 Waiver", + "metadataBlocks": { + "citation": { + "displayName": "Citation Metadata", + "fields": [ + { + "typeName": "title", + "multiple": false, + "typeClass": "primitive", + "value": "Replication Data for: Title" + }, + { + "typeName": "subtitle", + "multiple": false, + "typeClass": "primitive", + "value": "Subtitle" + }, + { + "typeName": "alternativeTitle", + "multiple": false, + "typeClass": "primitive", + "value": "Alternative Title" + }, + { + "typeName": "author", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastAuthor1, FirstAuthor1" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorAffiliation1" + } + }, + { + "authorName": { + "typeName": "authorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastAuthor2, FirstAuthor2" + }, + "authorAffiliation": { + "typeName": "authorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "AuthorAffiliation2" + } + } + ] + }, + { + "typeName": "datasetContact", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContact1, FirstContact1" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ContactAffiliation1" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "ContactEmail1@mailinator.com" + } + }, + { + "datasetContactName": { + "typeName": "datasetContactName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContact2, FirstContact2" + }, + "datasetContactAffiliation": { + "typeName": "datasetContactAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ContactAffiliation2" + }, + "datasetContactEmail": { + "typeName": "datasetContactEmail", + "multiple": false, + "typeClass": "primitive", + "value": "ContactEmail2@mailinator.com" + } + } + ] + }, + { + "typeName": "dsDescription", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "DescriptionText 1" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1000-01-01" + } + }, + { + "dsDescriptionValue": { + "typeName": "dsDescriptionValue", + "multiple": false, + "typeClass": "primitive", + "value": "DescriptionText2" + }, + "dsDescriptionDate": { + "typeName": "dsDescriptionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1000-02-02" + } + } + ] + }, + { + "typeName": "subject", + "multiple": true, + "typeClass": "controlledVocabulary", + "value": [ + "Agricultural Sciences", + "Business and Management", + "Engineering", + "Law" + ] + }, + { + "typeName": "keyword", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordTerm1" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordVocabulary1" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://KeywordVocabularyURL1.org" + } + }, + { + "keywordValue": { + "typeName": "keywordValue", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordTerm2" + }, + "keywordVocabulary": { + "typeName": "keywordVocabulary", + "multiple": false, + "typeClass": "primitive", + "value": "KeywordVocabulary2" + }, + "keywordVocabularyURI": { + "typeName": "keywordVocabularyURI", + "multiple": false, + "typeClass": "primitive", + "value": "http://KeywordVocabularyURL2.org" + } + } + ] + }, + { + "typeName": "publication", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation1" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "ark" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber1" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL1.org" + } + }, + { + "publicationCitation": { + "typeName": "publicationCitation", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationCitation2" + }, + "publicationIDType": { + "typeName": "publicationIDType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "arXiv" + }, + "publicationIDNumber": { + "typeName": "publicationIDNumber", + "multiple": false, + "typeClass": "primitive", + "value": "RelatedPublicationIDNumber2" + }, + "publicationURL": { + "typeName": "publicationURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://RelatedPublicationURL2.org" + } + } + ] + }, + { + "typeName": "notesText", + "multiple": false, + "typeClass": "primitive", + "value": "Notes1" + }, + { + "typeName": "producer", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "LastProducer1, FirstProducer1" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAffiliation1" + }, + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAbbreviation1" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerURL1.org" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerLogoURL1.org" + } + }, + { + "producerName": { + "typeName": "producerName", + "multiple": false, + "typeClass": "primitive", + "value": "LastProducer2, FirstProducer2" + }, + "producerAffiliation": { + "typeName": "producerAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAffiliation2" + }, + "producerAbbreviation": { + "typeName": "producerAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "ProducerAbbreviation2" + }, + "producerURL": { + "typeName": "producerURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerURL2.org" + }, + "producerLogoURL": { + "typeName": "producerLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://ProducerLogoURL2.org" + } + } + ] + }, + { + "typeName": "productionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1003-01-01" + }, + { + "typeName": "productionPlace", + "multiple": false, + "typeClass": "primitive", + "value": "ProductionPlace" + }, + { + "typeName": "grantNumber", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "grantNumberAgency": { + "typeName": "grantNumberAgency", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantAgency1" + }, + "grantNumberValue": { + "typeName": "grantNumberValue", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantNumber1" + } + }, + { + "grantNumberAgency": { + "typeName": "grantNumberAgency", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantAgency2" + }, + "grantNumberValue": { + "typeName": "grantNumberValue", + "multiple": false, + "typeClass": "primitive", + "value": "GrantInformationGrantNumber2" + } + } + ] + }, + { + "typeName": "distributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "distributorName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastDistributor1, FirstDistributor1" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAffiliation1" + }, + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAbbreviation1" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorURL1.org" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorLogoURL1.org" + } + }, + { + "distributorName": { + "typeName": "distributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastDistributor2, FirstDistributor2" + }, + "distributorAffiliation": { + "typeName": "distributorAffiliation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAffiliation2" + }, + "distributorAbbreviation": { + "typeName": "distributorAbbreviation", + "multiple": false, + "typeClass": "primitive", + "value": "DistributorAbbreviation2" + }, + "distributorURL": { + "typeName": "distributorURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorURL2.org" + }, + "distributorLogoURL": { + "typeName": "distributorLogoURL", + "multiple": false, + "typeClass": "primitive", + "value": "http://DistributorLogoURL2.org" + } + } + ] + }, + { + "typeName": "distributionDate", + "multiple": false, + "typeClass": "primitive", + "value": "1004-01-01" + }, + { + "typeName": "depositor", + "multiple": false, + "typeClass": "primitive", + "value": "LastDepositor, FirstDepositor" + }, + { + "typeName": "dateOfDeposit", + "multiple": false, + "typeClass": "primitive", + "value": "1002-01-01" + }, + { + "typeName": "timePeriodCovered", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "timePeriodCoveredStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "1005-01-01" + }, + "timePeriodCoveredEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1005-01-02" + } + }, + { + "timePeriodCoveredStart": { + "typeName": "timePeriodCoveredStart", + "multiple": false, + "typeClass": "primitive", + "value": "1005-02-01" + }, + "timePeriodCoveredEnd": { + "typeName": "timePeriodCoveredEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1005-02-02" + } + } + ] + }, + { + "typeName": "dateOfCollection", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "dateOfCollectionStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "1006-01-01" + }, + "dateOfCollectionEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1006-01-01" + } + }, + { + "dateOfCollectionStart": { + "typeName": "dateOfCollectionStart", + "multiple": false, + "typeClass": "primitive", + "value": "1006-02-01" + }, + "dateOfCollectionEnd": { + "typeName": "dateOfCollectionEnd", + "multiple": false, + "typeClass": "primitive", + "value": "1006-02-02" + } + } + ] + }, + { + "typeName": "kindOfData", + "multiple": true, + "typeClass": "primitive", + "value": [ + "KindOfData1", + "KindOfData2" + ] + }, + { + "typeName": "series", + "multiple": false, + "typeClass": "compound", + "value": { + "seriesName": { + "typeName": "seriesName", + "multiple": false, + "typeClass": "primitive", + "value": "SeriesName" + }, + "seriesInformation": { + "typeName": "seriesInformation", + "multiple": false, + "typeClass": "primitive", + "value": "SeriesInformation" + } + } + }, + { + "typeName": "software", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "softwareName": { + "typeName": "softwareName", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareName1" + }, + "softwareVersion": { + "typeName": "softwareVersion", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareVersion1" + } + }, + { + "softwareName": { + "typeName": "softwareName", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareName2" + }, + "softwareVersion": { + "typeName": "softwareVersion", + "multiple": false, + "typeClass": "primitive", + "value": "SoftwareVersion2" + } + } + ] + }, + { + "typeName": "relatedMaterial", + "multiple": true, + "typeClass": "primitive", + "value": [ + "RelatedMaterial1", + "RelatedMaterial2" + ] + }, + { + "typeName": "relatedDatasets", + "multiple": true, + "typeClass": "primitive", + "value": [ + "RelatedDatasets1", + "RelatedDatasets2" + ] + }, + { + "typeName": "otherReferences", + "multiple": true, + "typeClass": "primitive", + "value": [ + "OtherReferences1", + "OtherReferences2" + ] + }, + { + "typeName": "dataSources", + "multiple": true, + "typeClass": "primitive", + "value": [ + "DataSources1", + "DataSources2" + ] + }, + { + "typeName": "originOfSources", + "multiple": false, + "typeClass": "primitive", + "value": "OriginOfSources" + }, + { + "typeName": "characteristicOfSources", + "multiple": false, + "typeClass": "primitive", + "value": "CharacteristicOfSourcesNoted" + }, + { + "typeName": "accessToSources", + "multiple": false, + "typeClass": "primitive", + "value": "DocumentationAndAccessToSources" + } + ] + }, + "geospatial": { + "displayName": "Geospatial Metadata", + "fields": [ + { + "typeName": "geographicCoverage", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Afghanistan" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageStateProvince1" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageCity1" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageOther1" + } + }, + { + "country": { + "typeName": "country", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Albania" + }, + "state": { + "typeName": "state", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageStateProvince2" + }, + "city": { + "typeName": "city", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageCity2" + }, + "otherGeographicCoverage": { + "typeName": "otherGeographicCoverage", + "multiple": false, + "typeClass": "primitive", + "value": "GeographicCoverageOther2" + } + } + ] + }, + { + "typeName": "geographicUnit", + "multiple": true, + "typeClass": "primitive", + "value": [ + "GeographicUnit1", + "GeographicUnit2" + ] + }, + { + "typeName": "geographicBoundingBox", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "10" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "20" + }, + "northLongitude": { + "typeName": "northLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "30" + }, + "southLongitude": { + "typeName": "southLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "40" + } + }, + { + "westLongitude": { + "typeName": "westLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "50" + }, + "eastLongitude": { + "typeName": "eastLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "60" + }, + "northLongitude": { + "typeName": "northLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "70" + }, + "southLongitude": { + "typeName": "southLongitude", + "multiple": false, + "typeClass": "primitive", + "value": "80" + } + } + ] + } + ] + }, + "socialscience": { + "displayName": "Social Science and Humanities Metadata", + "fields": [ + { + "typeName": "unitOfAnalysis", + "multiple": true, + "typeClass": "primitive", + "value": [ + "UnitOfAnalysis1", + "UnitOfAnalysis2" + ] + }, + { + "typeName": "universe", + "multiple": true, + "typeClass": "primitive", + "value": [ + "Universe1", + "Universe2" + ] + }, + { + "typeName": "timeMethod", + "multiple": false, + "typeClass": "primitive", + "value": "TimeMethod" + }, + { + "typeName": "dataCollector", + "multiple": false, + "typeClass": "primitive", + "value": "LastDataCollector1, FirstDataCollector1" + }, + { + "typeName": "collectorTraining", + "multiple": false, + "typeClass": "primitive", + "value": "CollectorTraining" + }, + { + "typeName": "frequencyOfDataCollection", + "multiple": false, + "typeClass": "primitive", + "value": "Frequency" + }, + { + "typeName": "samplingProcedure", + "multiple": false, + "typeClass": "primitive", + "value": "SamplingProcedure" + }, + { + "typeName": "targetSampleSize", + "multiple": false, + "typeClass": "compound", + "value": { + "targetSampleActualSize": { + "typeName": "targetSampleActualSize", + "multiple": false, + "typeClass": "primitive", + "value": "100" + }, + "targetSampleSizeFormula": { + "typeName": "targetSampleSizeFormula", + "multiple": false, + "typeClass": "primitive", + "value": "TargetSampleSizeFormula" + } + } + }, + { + "typeName": "deviationsFromSampleDesign", + "multiple": false, + "typeClass": "primitive", + "value": "MajorDeviationsForSampleDesign" + }, + { + "typeName": "collectionMode", + "multiple": false, + "typeClass": "primitive", + "value": "CollectionMode" + }, + { + "typeName": "researchInstrument", + "multiple": false, + "typeClass": "primitive", + "value": "TypeOfResearchInstrument" + }, + { + "typeName": "dataCollectionSituation", + "multiple": false, + "typeClass": "primitive", + "value": "CharacteristicsOfDataCollectionSituation" + }, + { + "typeName": "actionsToMinimizeLoss", + "multiple": false, + "typeClass": "primitive", + "value": "ActionsToMinimizeLosses" + }, + { + "typeName": "controlOperations", + "multiple": false, + "typeClass": "primitive", + "value": "ControlOperations" + }, + { + "typeName": "weighting", + "multiple": false, + "typeClass": "primitive", + "value": "Weighting" + }, + { + "typeName": "cleaningOperations", + "multiple": false, + "typeClass": "primitive", + "value": "CleaningOperations" + }, + { + "typeName": "datasetLevelErrorNotes", + "multiple": false, + "typeClass": "primitive", + "value": "StudyLevelErrorNotes" + }, + { + "typeName": "responseRate", + "multiple": false, + "typeClass": "primitive", + "value": "ResponseRate" + }, + { + "typeName": "samplingErrorEstimates", + "multiple": false, + "typeClass": "primitive", + "value": "EstimatesOfSamplingError" + }, + { + "typeName": "otherDataAppraisal", + "multiple": false, + "typeClass": "primitive", + "value": "OtherFormsOfDataAppraisal" + }, + { + "typeName": "socialScienceNotes", + "multiple": false, + "typeClass": "compound", + "value": { + "socialScienceNotesType": { + "typeName": "socialScienceNotesType", + "multiple": false, + "typeClass": "primitive", + "value": "NotesType" + }, + "socialScienceNotesSubject": { + "typeName": "socialScienceNotesSubject", + "multiple": false, + "typeClass": "primitive", + "value": "NotesSubject" + }, + "socialScienceNotesText": { + "typeName": "socialScienceNotesText", + "multiple": false, + "typeClass": "primitive", + "value": "NotesText" + } + } + } + ] + } + }, + "files": [], + "citation": "LastAuthor1, FirstAuthor1; LastAuthor2, FirstAuthor2, 2020, \"Replication Data for: Title\", https://doi.org/10.5072/FK2/WKUKGV, Root, V1" + } +} diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml new file mode 100644 index 00000000000..390f1968d9a --- /dev/null +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -0,0 +1,156 @@ + + + + + + Replication Data for: Title + doi:10.5072/FK2/WKUKGV + + + Root + 2020-02-19 + + + 1 + + LastAuthor1, FirstAuthor1; LastAuthor2, FirstAuthor2, 2020, "Replication Data for: Title", https://doi.org/10.5072/FK2/WKUKGV, Root, V1 + + + + + + Replication Data for: Title + Subtitle + Alternative Title + doi:10.5072/FK2/WKUKGV + + + LastAuthor1, FirstAuthor1 + LastAuthor2, FirstAuthor2 + + + LastProducer1, FirstProducer1 + LastProducer2, FirstProducer2 + 1003-01-01 + ProductionPlace + SoftwareName1 + SoftwareName2 + GrantInformationGrantNumber1 + GrantInformationGrantNumber2 + + + LastDistributor1, FirstDistributor1 + LastDistributor2, FirstDistributor2 + LastContact1, FirstContact1 + LastContact2, FirstContact2 + 1004-01-01 + LastDepositor, FirstDepositor + 1002-01-01 + + + SeriesName + SeriesInformation + + + + + Agricultural Sciences + Business and Management + Engineering + Law + KeywordTerm1 + KeywordTerm2 + + DescriptionText 1 + DescriptionText2 + + 1005-01-01 + 1005-01-02 + 1005-02-01 + 1005-02-02 + 1006-01-01 + 1006-01-01 + 1006-02-01 + 1006-02-02 + KindOfData1 + KindOfData2 + Afghanistan + GeographicCoverageCity1 + GeographicCoverageStateProvince1 + GeographicCoverageOther1 + Albania + GeographicCoverageCity2 + GeographicCoverageStateProvince2 + GeographicCoverageOther2 + + 10 + 20 + 30 + 40 + + + 80 + 70 + 60 + 50 + + GeographicUnit1 + GeographicUnit2 + UnitOfAnalysis1 + UnitOfAnalysis2 + Universe1 + Universe2 + + Notes1 + + + + TimeMethod + LastDataCollector1, FirstDataCollector1 + CollectorTraining + Frequency + SamplingProcedure + + TargetSampleSizeFormula + 100 + + MajorDeviationsForSampleDesign + + DataSources1 + DataSources2 + OriginOfSources + CharacteristicOfSourcesNoted + DocumentationAndAccessToSources + + CollectionMode + TypeOfResearchInstrument + CharacteristicsOfDataCollectionSituation + ActionsToMinimizeLosses + ControlOperations + Weighting + CleaningOperations + + + ResponseRate + EstimatesOfSamplingError + OtherFormsOfDataAppraisal + + NotesText + + + CC0 Waiver + + + + RelatedMaterial1 + RelatedMaterial2 + RelatedDatasets1 + RelatedDatasets2 + RelatedPublicationCitation1, ark, RelatedPublicationIDNumber1, http://RelatedPublicationURL1.org + RelatedPublicationCitation2, arXiv, RelatedPublicationIDNumber2, http://RelatedPublicationURL2.org + OtherReferences1 + OtherReferences2 + + StudyLevelErrorNotes + + From 03d48a8568bec89d2b7757df66fc1f4b16fccc8c Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Fri, 21 Feb 2020 14:29:55 -0500 Subject: [PATCH 096/157] remove commented lines --- .../harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 528070c31d6..03179f1e29d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -779,7 +779,6 @@ private void processSumDscr(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) throw geoCoverageSet.add(FieldDTO.createPrimitiveFieldDTO("otherGeographicCoverage", otherGeographicCoverage)); } if (geoCoverageSet != null && geoCoverageSet.size() > 0) { - //FieldDTO geoCoverageDTO = FieldDTO.createMultipleCompoundFieldDTO(DatasetFieldConstant.geographicCoverage, geoCoverageList); geoCoverages.add(geoCoverageSet); } if (geoCoverages.size() > 0) { @@ -826,8 +825,6 @@ private void processMethod(XMLStreamReader xmlr, DatasetVersionDTO dvDTO ) throw processCustomField(xmlr, dvDTO); } else { processNotesSocialScience(xmlr, dvDTO); -// addNote("Subject: Study Level Error Note, Notes: "+ parseText( xmlr,"notes" ) +";", dvDTO); - } } else if (xmlr.getLocalName().equals("anlyInfo")) { processAnlyInfo(xmlr, getSocialScience(dvDTO)); From ddfeb3e05edd25512165abfbbb3e28f9397f66de Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 21 Feb 2020 15:17:34 -0500 Subject: [PATCH 097/157] Disallow 'placeholder' storageIO ~clones --- .../harvard/iq/dataverse/dataaccess/DataAccess.java | 13 +++++++++++-- .../harvard/iq/dataverse/dataset/DatasetUtil.java | 2 +- .../harvard/iq/dataverse/export/ExportService.java | 2 +- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index f5ec98f14be..907a2173138 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -63,7 +63,7 @@ public static StorageIO getStorageIO(T dvObject, DataAcc } String storageIdentifier = dvObject.getStorageIdentifier(); int separatorIndex = storageIdentifier.indexOf("://"); - String storageDriverId = "file"; //default + String storageDriverId = DEFAULT_STORAGE_DRIVER_IDENTIFIER; //default if(separatorIndex>0) { storageDriverId = storageIdentifier.substring(0,separatorIndex); } @@ -142,6 +142,7 @@ public static StorageIO createNewStorageIO(T dvObject, S || storageTag.isEmpty()) { throw new IOException("getDataAccessObject: null or invalid datafile."); } + return createNewStorageIO(dvObject, storageTag, dvObject.getDataverseContext().getEffectiveStorageDriverId()); } @@ -151,13 +152,21 @@ public static StorageIO createNewStorageIO(T dvObject, S || storageTag.isEmpty()) { throw new IOException("getDataAccessObject: null or invalid datafile."); } + + /* Prior versions sometimes called createNewStorageIO(object, "placeholder") with an existing object to get a ~clone for use in storing/reading Aux files + * Since PR #6488 for multi-store - this can return a clone using a different store than the original (e.g. if the default store changes) which causes errors + * This if will catch any cases where that's attempted. + */ + if(dvObject.getStorageIdentifier().contains("://")) { + throw new IOException("Attempt to create new StorageIO for already stored object: " + dvObject.getStorageIdentifier()); + } StorageIO storageIO = null; dvObject.setStorageIdentifier(storageTag); if (StringUtils.isEmpty(storageDriverId)) { - storageDriverId = "file"; + storageDriverId = DEFAULT_STORAGE_DRIVER_IDENTIFIER; } String storageType = getDriverType(storageDriverId); switch(storageType) { diff --git a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java index b844a9d109f..a5931da26f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataset/DatasetUtil.java @@ -274,7 +274,7 @@ public static Dataset persistDatasetLogoToStorageAndCreateThumbnail(Dataset data StorageIO dataAccess = null; try{ - dataAccess = DataAccess.createNewStorageIO(dataset,"placeholder"); + dataAccess = DataAccess.getStorageIO(dataset); } catch(IOException ioex){ //TODO: Add a suitable waing message diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index e61efca2ddf..122a0e482bf 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -253,7 +253,7 @@ private void cacheExport(DatasetVersion version, String format, JsonObject datas // to save the output into a temp file, and then copy it over to the // permanent storage using the IO "save" command: try { - storageIO = DataAccess.createNewStorageIO(dataset, "placeholder"); + storageIO = DataAccess.getStorageIO(dataset); Channel outputChannel = storageIO.openAuxChannel("export_" + format + ".cached", DataAccessOption.WRITE_ACCESS); outputStream = Channels.newOutputStream((WritableByteChannel) outputChannel); } catch (IOException ioex) { From de4c5d01ad6e85a9a515da96f59af7740eff4869 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 21 Feb 2020 15:41:04 -0500 Subject: [PATCH 098/157] bug fix for exporting plus some cleanup to let one exporter fail without stopping an overall loop through all formats. --- .../iq/dataverse/export/ExportService.java | 102 +++++++++--------- 1 file changed, 50 insertions(+), 52 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java index 122a0e482bf..9eaf0dbced0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ExportService.java @@ -240,58 +240,56 @@ public Exporter getExporter(String formatName) throws ExportException { // This method runs the selected metadata exporter, caching the output // in a file in the dataset directory / container based on its DOI: private void cacheExport(DatasetVersion version, String format, JsonObject datasetAsJson, Exporter exporter) throws ExportException { - boolean tempFileRequired = false; - File tempFile = null; - OutputStream outputStream = null; - Dataset dataset = version.getDataset(); - StorageIO storageIO = null; - try { - // With some storage drivers, we can open a WritableChannel, or OutputStream - // to directly write the generated metadata export that we want to cache; - // Some drivers (like Swift) do not support that, and will give us an - // "operation not supported" exception. If that's the case, we'll have - // to save the output into a temp file, and then copy it over to the - // permanent storage using the IO "save" command: - try { - storageIO = DataAccess.getStorageIO(dataset); - Channel outputChannel = storageIO.openAuxChannel("export_" + format + ".cached", DataAccessOption.WRITE_ACCESS); - outputStream = Channels.newOutputStream((WritableByteChannel) outputChannel); - } catch (IOException ioex) { - tempFileRequired = true; - tempFile = File.createTempFile("tempFileToExport", ".tmp"); - outputStream = new FileOutputStream(tempFile); - } - - try { - Path cachedMetadataFilePath = Paths.get(version.getDataset().getFileSystemDirectory().toString(), "export_" + format + ".cached"); - - if (!tempFileRequired) { - FileOutputStream cachedExportOutputStream = new FileOutputStream(cachedMetadataFilePath.toFile()); - exporter.exportDataset(version, datasetAsJson, cachedExportOutputStream); - cachedExportOutputStream.flush(); - cachedExportOutputStream.close(); - outputStream.close(); - } else { - // this method copies a local filesystem Path into this DataAccess Auxiliary location: - exporter.exportDataset(version, datasetAsJson, outputStream); - outputStream.flush(); - outputStream.close(); - - logger.fine("Saving path as aux for temp file in: " + Paths.get(tempFile.getAbsolutePath())); - storageIO.savePathAsAux(Paths.get(tempFile.getAbsolutePath()), "export_" + format + ".cached"); - boolean tempFileDeleted = tempFile.delete(); - logger.fine("tempFileDeleted: " + tempFileDeleted); - } - - } catch (IOException ioex) { - throw new ExportException("IO Exception thrown exporting as " + "export_" + format + ".cached"); - } - - } catch (IOException ioex) { - throw new ExportException("IO Exception thrown exporting as " + "export_" + format + ".cached"); - } finally { - IOUtils.closeQuietly(outputStream); - } + boolean tempFileUsed = false; + File tempFile = null; + OutputStream outputStream = null; + Dataset dataset = version.getDataset(); + StorageIO storageIO = null; + try { + // With some storage drivers, we can open a WritableChannel, or OutputStream + // to directly write the generated metadata export that we want to cache; + // Some drivers (like Swift) do not support that, and will give us an + // "operation not supported" exception. If that's the case, we'll have + // to save the output into a temp file, and then copy it over to the + // permanent storage using the IO "save" command: + try { + storageIO = DataAccess.getStorageIO(dataset); + Channel outputChannel = storageIO.openAuxChannel("export_" + format + ".cached", DataAccessOption.WRITE_ACCESS); + outputStream = Channels.newOutputStream((WritableByteChannel) outputChannel); + } catch (IOException ioex) { + // A common case = an IOException in openAuxChannel which is not supported by S3 stores for WRITE_ACCESS + tempFileUsed = true; + tempFile = File.createTempFile("tempFileToExport", ".tmp"); + outputStream = new FileOutputStream(tempFile); + } + + try { + // Write the metadata export file to the outputStream, which may be the final location or a temp file + exporter.exportDataset(version, datasetAsJson, outputStream); + outputStream.flush(); + outputStream.close(); + if(tempFileUsed) { + logger.fine("Saving export_" + format + ".cached aux file from temp file: " + Paths.get(tempFile.getAbsolutePath())); + storageIO.savePathAsAux(Paths.get(tempFile.getAbsolutePath()), "export_" + format + ".cached"); + boolean tempFileDeleted = tempFile.delete(); + logger.fine("tempFileDeleted: " + tempFileDeleted); + } + } catch (ExportException exex) { + /*This exception is from the particular exporter and may not affect other exporters (versus other exceptions in this method which are from the basic mechanism to create a file) + * So we'll catch it here and report so that loops over other exporters can continue. + * Todo: Might be better to create a new exception subtype and send it upward, but the callers currently just log and ignore beyond terminating any loop over exporters. + */ + logger.warning("Exception thrown while creating export_" + format + ".cached : " + exex.getMessage()); + } catch (IOException ioex) { + throw new ExportException("IO Exception thrown exporting as " + "export_" + format + ".cached"); + } + + } catch (IOException ioex) { + //This catches any problem creating a local temp file in the catch clause above + throw new ExportException("IO Exception thrown before exporting as " + "export_" + format + ".cached"); + } finally { + IOUtils.closeQuietly(outputStream); + } } From 2e5009c64ccecceaad3affb2adfecc88da503424 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 24 Feb 2020 18:56:44 -0500 Subject: [PATCH 099/157] typo --- doc/sphinx-guides/source/developers/big-data-support.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index f1371f41759..8f1103cf720 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -23,7 +23,7 @@ To configure these options, an administrator must set two JVM options for the Da With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific dataverses) while configuring only direct download, or no direct access for another store. -At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by Dataverse from the S3 store). A store using direct uplod can be configured to disable all ingest processing for files above a given size limit: +At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by Dataverse from the S3 store). A store using direct upload can be configured to disable all ingest processing for files above a given size limit: ``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="`` From e6aee06c8d5742053af275f244eb10c9e6f70013 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 26 Feb 2020 09:38:52 -0500 Subject: [PATCH 100/157] get the effective storage driver --- src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 2 +- src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 5ec37030024..681c9c15dd9 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -358,7 +358,7 @@ public boolean doesSessionUserHaveDataSetPermission(Permission permissionToCheck } public boolean directUploadEnabled() { - return Boolean.getBoolean("dataverse.files." + this.dataset.getDataverseContext().getStorageDriverId() + ".upload-redirect"); + return Boolean.getBoolean("dataverse.files." + this.dataset.getDataverseContext().getEffectiveStorageDriverId() + ".upload-redirect"); } public void reset() { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 99ff6994db2..820a7037605 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1650,7 +1650,7 @@ public static boolean isPackageFile(DataFile dataFile) { } public static S3AccessIO getS3AccessForDirectUpload(Dataset dataset) { - String driverId = dataset.getDataverseContext().getStorageDriverId(); + String driverId = dataset.getDataverseContext().getEffectiveStorageDriverId(); boolean directEnabled = Boolean.getBoolean("dataverse.files." + driverId + ".upload-redirect"); //Should only be requested when it is allowed, but we'll log a warning otherwise if(!directEnabled) { From 91f42755ba488ae73155b4321a53635b59377058 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Mon, 2 Mar 2020 14:39:41 -0500 Subject: [PATCH 101/157] don't wrap file --- src/main/webapp/resources/js/fileupload.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index c3da541414a..83d42b9f417 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -45,16 +45,14 @@ function queueFileForDirectUpload(file, datasetId) { } function uploadFileDirectly(url, storageId) { - var data = new FormData(); //Pick a pending file var file = fileList.pop(); - data.append('file',file); $('.ui-fileupload-progress').html(''); $('.ui-fileupload-progress').append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); $.ajax({ url: url, type: 'PUT', - data: data, + data: file, cache: false, processData: false, success: function () { From 8446bfb5deae3ae2e106801f5786ace907462198 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Tue, 3 Mar 2020 12:41:22 -0500 Subject: [PATCH 102/157] fix export import ddi --- .../api/imports/ImportDDIServiceBean.java | 131 ++++++++++++------ .../dataverse/export/ddi/DdiExportUtil.java | 94 ++++++++++--- .../dataset-create-new-all-ddi-fields.json | 84 +++++++++++ .../dataverse/export/ddi/dataset-finch1.xml | 1 + .../iq/dataverse/export/ddi/exportfull.xml | 46 +++++- 5 files changed, 288 insertions(+), 68 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java index 03179f1e29d..ce2f646322c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/imports/ImportDDIServiceBean.java @@ -10,6 +10,8 @@ import edu.harvard.iq.dataverse.api.dto.MetadataBlockDTO; import edu.harvard.iq.dataverse.api.imports.ImportUtil.ImportType; import static edu.harvard.iq.dataverse.export.ddi.DdiExportUtil.NOTE_TYPE_CONTENTTYPE; +import static edu.harvard.iq.dataverse.export.ddi.DdiExportUtil.NOTE_TYPE_TERMS_OF_ACCESS; + import edu.harvard.iq.dataverse.util.StringUtil; import java.io.File; import java.io.FileInputStream; @@ -267,18 +269,18 @@ private void processCodeBook(ImportType importType, XMLStreamReader xmlr, Datase } } } - + private void processDocDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException { for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { - + if (xmlr.getLocalName().equals("IDNo") && StringUtil.isEmpty(datasetDTO.getIdentifier()) ) { // this will set a StudyId if it has not yet been set; it will get overridden by a metadata // id in the StudyDscr section, if one exists if ( AGENCY_HANDLE.equals( xmlr.getAttributeValue(null, "agency") ) ) { parseStudyIdHandle( parseText(xmlr), datasetDTO ); - } - // EMK TODO: we need to save this somewhere when we add harvesting infrastructure + } + // EMK TODO: we need to save this somewhere when we add harvesting infrastructure } /*else if ( xmlr.getLocalName().equals("holdings") && StringUtil.isEmpty(datasetDTO..getHarvestHoldings()) ) { metadata.setHarvestHoldings( xmlr.getAttributeValue(null, "URI") ); }*/ @@ -286,7 +288,7 @@ private void processDocDscr(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws if (xmlr.getLocalName().equals("docDscr")) return; } } - } + } private String parseText(XMLStreamReader xmlr) throws XMLStreamException { return parseText(xmlr,true); } @@ -393,41 +395,12 @@ else if (xmlr.getLocalName().equals("relStdy")) { relStudy.add(parseText(xmlr, "relStdy")); getCitation(dvDTO).addField(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.relatedDatasets, relStudy)); } else if (xmlr.getLocalName().equals("relPubl")) { - HashSet set = new HashSet<>(); - - // call new parse text logic - Object rpFromDDI = parseTextNew(xmlr, "relPubl"); - if (rpFromDDI instanceof Map) { - Map rpMap = (Map) rpFromDDI; - addToSet(set, DatasetFieldConstant.publicationCitation, rpMap.get("text")); - addToSet(set, DatasetFieldConstant.publicationIDNumber, rpMap.get("idNumber")); - addToSet(set, DatasetFieldConstant.publicationURL, rpMap.get("url")); - if (rpMap.get("idType")!=null) { - set.add(FieldDTO.createVocabFieldDTO(DatasetFieldConstant.publicationIDType, rpMap.get("idType").toLowerCase())); - } - // rp.setText((String) rpMap.get("text")); - // rp.setIdType((String) rpMap.get("idType")); - // rp.setIdNumber((String) rpMap.get("idNumber")); - // rp.setUrl((String) rpMap.get("url")); - // TODO: ask about where/whether we want to save this - // if (!replicationForFound && rpMap.get("replicationData") != null) { - // rp.setReplicationData(true); - /// replicationForFound = true; - // } - } else { - addToSet(set, DatasetFieldConstant.publicationCitation, (String) rpFromDDI); - // rp.setText( (String) rpFromDDI ); - } - publications.add(set); - - - } else if (xmlr.getLocalName().equals("othRefs")) { - + processRelPubl(xmlr, dvDTO, publications); + } else if (xmlr.getLocalName().equals("othRefs")) { List otherRefs = new ArrayList<>(); otherRefs.add(parseText(xmlr, "othRefs")); getCitation(dvDTO).addField(FieldDTO.createMultiplePrimitiveFieldDTO(DatasetFieldConstant.otherReferences, otherRefs)); - - } + } } else if (event == XMLStreamConstants.END_ELEMENT) { if (publications.size()>0) { getCitation(dvDTO).addField(FieldDTO.createMultipleCompoundFieldDTO(DatasetFieldConstant.publication, publications)); @@ -438,6 +411,49 @@ else if (xmlr.getLocalName().equals("relStdy")) { } } } + private void processRelPubl(XMLStreamReader xmlr, DatasetVersionDTO dvDTO, List> publications) throws XMLStreamException { + HashSet set = new HashSet<>(); + for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { + if (event == XMLStreamConstants.START_ELEMENT) { + if (xmlr.getLocalName().equals("citation")) { + for (int event2 = xmlr.next(); event2 != XMLStreamConstants.END_DOCUMENT; event2 = xmlr.next()) { + if (event2 == XMLStreamConstants.START_ELEMENT) { + if (xmlr.getLocalName().equals("titlStmt")) { + int event3 = xmlr.next(); + if (event3 == XMLStreamConstants.START_ELEMENT) { + if (xmlr.getLocalName().equals("IDNo")) { + set.add(FieldDTO.createVocabFieldDTO(DatasetFieldConstant.publicationIDType, xmlr.getAttributeValue(null, "agency"))); + addToSet(set, DatasetFieldConstant.publicationIDNumber, parseText(xmlr)); + } + } + } else if (xmlr.getLocalName().equals("biblCit")) { + if (event2 == XMLStreamConstants.START_ELEMENT) { + if (xmlr.getLocalName().equals("biblCit")) { + addToSet(set, DatasetFieldConstant.publicationCitation, parseText(xmlr)); + } + } + } + } else if (event2 == XMLStreamConstants.END_ELEMENT) { + if (xmlr.getLocalName().equals("citation")) { + break; + } + } + + } + } else if (xmlr.getLocalName().equals("ExtLink")) { + addToSet(set, DatasetFieldConstant.publicationURL, xmlr.getAttributeValue(null, "URI")); + } + } else if (event == XMLStreamConstants.END_ELEMENT) { + if (xmlr.getLocalName().equals("relPubl")) { + if (set.size() > 0) { + publications.add(set); + } + return; + } + } + } + } + private void processCitation(ImportType importType, XMLStreamReader xmlr, DatasetDTO datasetDTO) throws XMLStreamException, ImportException { DatasetVersionDTO dvDTO = datasetDTO.getDatasetVersion(); MetadataBlockDTO citation=datasetDTO.getDatasetVersion().getMetadataBlocks().get("citation"); @@ -1159,7 +1175,16 @@ private void processDataAccs(XMLStreamReader xmlr, DatasetVersionDTO dvDTO) thro String noteType = xmlr.getAttributeValue(null, "type"); if (NOTE_TYPE_TERMS_OF_USE.equalsIgnoreCase(noteType) ) { if ( LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) { - dvDTO.setTermsOfUse(parseText(xmlr, "notes")); + String termOfUse = parseText(xmlr, "notes"); + if (termOfUse != null && termOfUse.trim().equals("CC0 Waiver") ) { + dvDTO.setLicense("CC0"); + } else if (termOfUse != null && !termOfUse.trim().equals("")){ + dvDTO.setTermsOfUse(termOfUse); + } + } + } else if (NOTE_TYPE_TERMS_OF_ACCESS.equalsIgnoreCase(noteType) ) { + if (LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) { + dvDTO.setTermsOfAccess(parseText(xmlr, "notes")); } } else { processNotes(xmlr, dvDTO); @@ -1266,13 +1291,16 @@ private void processDistStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) th for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("distrbtr")) { - HashSet set = new HashSet<>(); - addToSet(set, "distributorAbbreviation", xmlr.getAttributeValue(null, "abbr")); - addToSet(set, "distributorAffiliation", xmlr.getAttributeValue(null, "affiliation")); - addToSet(set, "distributorURL", xmlr.getAttributeValue(null, "URI")); - addToSet(set, "distributorLogoURL", xmlr.getAttributeValue(null, "role")); - addToSet(set, "distributorName", xmlr.getElementText()); - distributors.add(set); + String source = xmlr.getAttributeValue(null, "source"); + if (source == null || !source.equals("archive")) { + HashSet set = new HashSet<>(); + addToSet(set, "distributorAbbreviation", xmlr.getAttributeValue(null, "abbr")); + addToSet(set, "distributorAffiliation", xmlr.getAttributeValue(null, "affiliation")); + addToSet(set, "distributorURL", xmlr.getAttributeValue(null, "URI")); + addToSet(set, "distributorLogoURL", xmlr.getAttributeValue(null, "role")); + addToSet(set, "distributorName", xmlr.getElementText()); + distributors.add(set); + } } else if (xmlr.getLocalName().equals("contact")) { HashSet set = new HashSet<>(); @@ -1414,6 +1442,7 @@ private void processTitlStmt(XMLStreamReader xmlr, DatasetDTO datasetDTO) throws private void processRspStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) throws XMLStreamException { List> authors = new ArrayList<>(); + List> contributors = new ArrayList<>(); for (int event = xmlr.next(); event != XMLStreamConstants.END_DOCUMENT; event = xmlr.next()) { if (event == XMLStreamConstants.START_ELEMENT) { if (xmlr.getLocalName().equals("AuthEnty")) { @@ -1424,12 +1453,24 @@ private void processRspStmt(XMLStreamReader xmlr, MetadataBlockDTO citation) thr authors.add(set); } } + if (xmlr.getLocalName().equals("othId")) { + HashSet set = new HashSet<>(); + set.add(FieldDTO.createVocabFieldDTO("contributorType", xmlr.getAttributeValue(null, "role") )); + addToSet(set,"contributorName", parseText(xmlr)); + if (!set.isEmpty()) { + contributors.add(set); + } + } } else if (event == XMLStreamConstants.END_ELEMENT) { if (xmlr.getLocalName().equals("rspStmt")) { if (authors.size()>0) { FieldDTO author = FieldDTO.createMultipleCompoundFieldDTO("author", authors); citation.getFields().add(author); } + if (contributors.size() > 0) { + FieldDTO contributor = FieldDTO.createMultipleCompoundFieldDTO("contributor", contributors); + citation.getFields().add(contributor); + } return; } diff --git a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java index db019d5a39a..a29d6ac7fbc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/export/ddi/DdiExportUtil.java @@ -73,7 +73,9 @@ public class DdiExportUtil { private static final Logger logger = Logger.getLogger(DdiExportUtil.class.getCanonicalName()); public static final String NOTE_TYPE_TERMS_OF_USE = "DVN:TOU"; - public static final String NOTE_SUBJECT_TERMS_OF_USE = "Terms Of Use"; + public static final String NOTE_TYPE_TERMS_OF_ACCESS = "DVN:TOA"; + public static final String NOTE_TYPE_DATA_ACCESS_PLACE = "DVN:DAP"; + public static final String LEVEL_DV = "dv"; @@ -186,13 +188,19 @@ private static void createStdyDscr(XMLStreamWriter xmlw, DatasetDTO datasetDto) writeAttribute(xmlw, "agency", persistentAgency); xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + "/" + persistentId); xmlw.writeEndElement(); // IDNo - + writeOtherIdElement(xmlw, version); xmlw.writeEndElement(); // titlStmt writeAuthorsElement(xmlw, version); writeProducersElement(xmlw, version); xmlw.writeStartElement("distStmt"); + if (datasetDto.getPublisher() != null && !datasetDto.getPublisher().equals("")) { + xmlw.writeStartElement("distrbtr"); + writeAttribute(xmlw, "source", "archive"); + xmlw.writeCharacters(datasetDto.getPublisher()); + xmlw.writeEndElement(); //distrbtr + } writeDistributorsElement(xmlw, version); writeContactsElement(xmlw, version); writeFullElement(xmlw, "distDate", dto2Primitive(version, DatasetFieldConstant.distributionDate)); @@ -245,23 +253,27 @@ private static void writeDataAccess(XMLStreamWriter xmlw , DatasetVersionDTO ver xmlw.writeCharacters(version.getTermsOfUse()); xmlw.writeEndElement(); //notes } - /*else if (xmlr.getLocalName().equals("notes")) { - String noteType = xmlr.getAttributeValue(null, "type"); - if (NOTE_TYPE_TERMS_OF_USE.equalsIgnoreCase(noteType) ) { - if ( LEVEL_DV.equalsIgnoreCase(xmlr.getAttributeValue(null, "level"))) { - dvDTO.setTermsOfUse(parseText(xmlr, "notes")); - }*/ + if (version.getTermsOfAccess() != null && !version.getTermsOfAccess().trim().equals("")) { + xmlw.writeStartElement("notes"); + writeAttribute(xmlw, "type", NOTE_TYPE_TERMS_OF_ACCESS); + writeAttribute(xmlw, "level", LEVEL_DV); + xmlw.writeCharacters(version.getTermsOfAccess()); + xmlw.writeEndElement(); //notes + } + + xmlw.writeStartElement("setAvail"); writeFullElement(xmlw, "accsPlac", version.getDataAccessPlace()); writeFullElement(xmlw, "origArch", version.getOriginalArchive()); writeFullElement(xmlw, "avlStatus", version.getAvailabilityStatus()); writeFullElement(xmlw, "collSize", version.getSizeOfCollection()); writeFullElement(xmlw, "complete", version.getStudyCompletion()); + xmlw.writeEndElement(); //setAvail xmlw.writeStartElement("useStmt"); writeFullElement(xmlw, "confDec", version.getConfidentialityDeclaration()); writeFullElement(xmlw, "specPerm", version.getSpecialPermissions()); writeFullElement(xmlw, "restrctn", version.getRestrictions()); writeFullElement(xmlw, "contact", version.getContactForAccess()); - writeFullElement(xmlw, "citeReq", version.getCitationRequirements()); + writeFullElement(xmlw, "citReq", version.getCitationRequirements()); writeFullElement(xmlw, "deposReq", version.getDepositorRequirements()); writeFullElement(xmlw, "conditions", version.getConditions()); writeFullElement(xmlw, "disclaimer", version.getDisclaimer()); @@ -293,10 +305,15 @@ private static void writeDocDescElement (XMLStreamWriter xmlw, DatasetDTO datase xmlw.writeStartElement("IDNo"); writeAttribute(xmlw, "agency", persistentAgency); xmlw.writeCharacters(persistentProtocol + ":" + persistentAuthority + "/" + persistentId); - xmlw.writeEndElement(); // IDNo + xmlw.writeEndElement(); // IDNo xmlw.writeEndElement(); // titlStmt xmlw.writeStartElement("distStmt"); - writeFullElement(xmlw, "distrbtr", datasetDto.getPublisher()); + if (datasetDto.getPublisher() != null && !datasetDto.getPublisher().equals("")) { + xmlw.writeStartElement("distrbtr"); + writeAttribute(xmlw, "source", "archive"); + xmlw.writeCharacters(datasetDto.getPublisher()); + xmlw.writeEndElement(); // distrbtr + } writeFullElement(xmlw, "distDate", datasetDto.getPublicationDate()); xmlw.writeEndElement(); // diststmt @@ -606,9 +623,9 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO String key = entry.getKey(); MetadataBlockDTO value = entry.getValue(); if ("citation".equals(key)) { + xmlw.writeStartElement("rspStmt"); for (FieldDTO fieldDTO : value.getFields()) { if (DatasetFieldConstant.author.equals(fieldDTO.getTypeName())) { - xmlw.writeStartElement("rspStmt"); String authorName = ""; String authorAffiliation = ""; for (HashSet foo : fieldDTO.getMultipleCompound()) { @@ -630,10 +647,34 @@ private static void writeAuthorsElement(XMLStreamWriter xmlw, DatasetVersionDTO xmlw.writeEndElement(); //AuthEnty } } - xmlw.writeEndElement(); //rspStmt + + } else if (DatasetFieldConstant.contributor.equals(fieldDTO.getTypeName())) { + String contributorName = ""; + String contributorType = ""; + for (HashSet foo : fieldDTO.getMultipleCompound()) { + for (Iterator iterator = foo.iterator(); iterator.hasNext();) { + FieldDTO next = iterator.next(); + if (DatasetFieldConstant.contributorName.equals(next.getTypeName())) { + contributorName = next.getSinglePrimitive(); + } + if (DatasetFieldConstant.contributorType.equals(next.getTypeName())) { + contributorType = next.getSinglePrimitive(); + } + } + if (!contributorName.isEmpty()){ + xmlw.writeStartElement("othId"); + if(!contributorType.isEmpty()){ + writeAttribute(xmlw,"role", contributorType); + } + xmlw.writeCharacters(contributorName); + xmlw.writeEndElement(); //othId + } + } } } + xmlw.writeEndElement(); //rspStmt } + } } @@ -832,12 +873,27 @@ private static void writeRelPublElement(XMLStreamWriter xmlw, DatasetVersionDTO url = next.getSinglePrimitive(); } } - pubString = appendCommaSeparatedValue(citation, IDType); - pubString = appendCommaSeparatedValue(pubString, IDNo); - pubString = appendCommaSeparatedValue(pubString, url); - if (!pubString.isEmpty()){ - xmlw.writeStartElement("relPubl"); - xmlw.writeCharacters(pubString); + if (citation != null && !citation.trim().equals("")) { + xmlw.writeStartElement("relPubl"); + xmlw.writeStartElement("citation"); + if (IDNo != null && !IDNo.trim().equals("")) { + xmlw.writeStartElement("titlStmt"); + xmlw.writeStartElement("IDNo"); + if (IDType != null && !IDType.trim().equals("")) { + xmlw.writeAttribute("agency", IDType ); + } + xmlw.writeCharacters(IDNo); + xmlw.writeEndElement(); //IDNo + xmlw.writeEndElement(); // titlStmt + } + + writeFullElement(xmlw,"biblCit",citation); + xmlw.writeEndElement(); //citation + if (url != null && !url.trim().equals("") ) { + xmlw.writeStartElement("ExtLink"); + xmlw.writeAttribute("URI", url); + xmlw.writeEndElement(); //ExtLink + } xmlw.writeEndElement(); //relPubl } } diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json index 31d43eb5377..8930938d1af 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-create-new-all-ddi-fields.json @@ -17,6 +17,20 @@ "createTime": "2015-09-24T16:47:51Z", "license": "CC0", "termsOfUse": "CC0 Waiver", + "termsOfAccess": "Terms of Access", + "dataAccessPlace": "Data Access Place", + "originalArchive": "Original Archive", + "availabilityStatus": "Availability Status", + "sizeOfCollection": "Size of Collection", + "studyCompletion": "Study Completion", + "confidentialityDeclaration": "Confidentiality Declaration", + "specialPermissions": "Special Permissions", + "restrictions": "Restrictions", + "contactForAccess": "Contact for Access", + "citationRequirements": "Citation Requirements", + "depositorRequirements": "Depositor Requirements", + "conditions": "Conditions ", + "disclaimer": "Disclaimer", "metadataBlocks": { "citation": { "displayName": "Citation Metadata", @@ -39,6 +53,41 @@ "typeClass": "primitive", "value": "Alternative Title" }, + { + "typeName": "otherId", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "otherIdAgency": { + "typeName": "otherIdAgency", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDAgency1" + }, + "otherIdValue": { + "typeName": "otherIdValue", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDIdentifier1" + } + }, + { + "otherIdAgency": { + "typeName": "otherIdAgency", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDAgency2" + }, + "otherIdValue": { + "typeName": "otherIdValue", + "multiple": false, + "typeClass": "primitive", + "value": "OtherIDIdentifier2" + } + } + ] + }, { "typeName": "author", "multiple": true, @@ -362,6 +411,41 @@ "typeClass": "primitive", "value": "ProductionPlace" }, + { + "typeName": "contributor", + "multiple": true, + "typeClass": "compound", + "value": [ + { + "contributorType": { + "typeName": "contributorType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Data Collector" + }, + "contributorName": { + "typeName": "contributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContributor1, FirstContributor1" + } + }, + { + "contributorType": { + "typeName": "contributorType", + "multiple": false, + "typeClass": "controlledVocabulary", + "value": "Data Curator" + }, + "contributorName": { + "typeName": "contributorName", + "multiple": false, + "typeClass": "primitive", + "value": "LastContributor2, FirstContributor2" + } + } + ] + }, { "typeName": "grantNumber", "multiple": true, diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml index 3e9fa6b0561..79e9e363994 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/dataset-finch1.xml @@ -64,6 +64,7 @@ + diff --git a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml index 390f1968d9a..d9be1217fc9 100644 --- a/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml +++ b/src/test/java/edu/harvard/iq/dataverse/export/ddi/exportfull.xml @@ -7,7 +7,7 @@ doi:10.5072/FK2/WKUKGV - Root + Root 2020-02-19 @@ -23,10 +23,14 @@ Subtitle Alternative Title doi:10.5072/FK2/WKUKGV + OtherIDIdentifier1 + OtherIDIdentifier2 LastAuthor1, FirstAuthor1 LastAuthor2, FirstAuthor2 + LastContributor1, FirstContributor1 + LastContributor2, FirstContributor2 LastProducer1, FirstProducer1 @@ -39,6 +43,7 @@ GrantInformationGrantNumber2 + Root LastDistributor1, FirstDistributor1 LastDistributor2, FirstDistributor2 LastContact1, FirstContact1 @@ -139,15 +144,48 @@ CC0 Waiver - + Terms of Access + + Data Access Place + Original Archive + Availability Status + Size of Collection + Study Completion + + + Confidentiality Declaration + Special Permissions + Restrictions + Contact for Access + Citation Requirements + Depositor Requirements + Conditions + Disclaimer + RelatedMaterial1 RelatedMaterial2 RelatedDatasets1 RelatedDatasets2 - RelatedPublicationCitation1, ark, RelatedPublicationIDNumber1, http://RelatedPublicationURL1.org - RelatedPublicationCitation2, arXiv, RelatedPublicationIDNumber2, http://RelatedPublicationURL2.org + + + + RelatedPublicationIDNumber1 + + RelatedPublicationCitation1 + + + + + + + RelatedPublicationIDNumber2 + + RelatedPublicationCitation2 + + + OtherReferences1 OtherReferences2 From 0052eeff66ef7008bdf9ce98a35c4275bb95720a Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 10:28:25 -0500 Subject: [PATCH 103/157] give default selection a value Conflicts: src/main/webapp/dataverse.xhtml --- src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 6 +++++- src/main/java/edu/harvard/iq/dataverse/DataversePage.java | 2 +- src/main/webapp/dataverse.xhtml | 3 +-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 4b53937a87f..3774e7576d6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -778,6 +778,10 @@ public String getStorageDriverId() { } public void setStorageDriverId(String storageDriver) { - this.storageDriver = storageDriver; + if(storageDriver!=null&&storageDriver.equals("default")) { + this.storageDriver=null; + } else { + this.storageDriver = storageDriver; + } } } diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 12f398c9c7d..9684071fe1e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -1214,7 +1214,7 @@ public Set> getStorageDriverOptions() { HashMap drivers =new HashMap(); drivers.putAll(DataAccess.getStorageDriverLabels()); //Add an entry for the default (inherited from an ancestor or the system default) - drivers.put(getDefaultStorageDriverLabel(), ""); + drivers.put(getDefaultStorageDriverLabel(), "default"); return drivers.entrySet(); } diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index e42630bb07b..323d501ed82 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -147,8 +147,7 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.storage.title']}">

- - + From 48d176ea9142858e76f5970e44427dc8aeb7d4c0 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 10:48:39 -0500 Subject: [PATCH 104/157] typo --- src/main/webapp/dataverse.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 323d501ed82..8530b149001 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -147,7 +147,7 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.storage.title']}">
- + From 94abdda08298476a38936ea8592a06495971ddfc Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 11:39:14 -0500 Subject: [PATCH 105/157] use non-null indicator for default for reported dataverse.storageIdentifier value Also fix fromAncestor logic Conflicts: src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java --- .../java/edu/harvard/iq/dataverse/Dataverse.java | 5 ++++- .../edu/harvard/iq/dataverse/DataversePage.java | 15 ++++++++++----- .../java/edu/harvard/iq/dataverse/api/Admin.java | 2 +- .../iq/dataverse/dataaccess/DataAccess.java | 5 +++-- src/main/webapp/dataverse.xhtml | 2 +- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 3774e7576d6..0eea108c461 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -774,11 +774,14 @@ public String getEffectiveStorageDriverId() { public String getStorageDriverId() { + if(storageDriver==null) { + return DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER; + } return storageDriver; } public void setStorageDriverId(String storageDriver) { - if(storageDriver!=null&&storageDriver.equals("default")) { + if(storageDriver!=null&&storageDriver.equals(DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER)) { this.storageDriver=null; } else { this.storageDriver = storageDriver; diff --git a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java index 9684071fe1e..165c1759b5e 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataversePage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataversePage.java @@ -1214,19 +1214,24 @@ public Set> getStorageDriverOptions() { HashMap drivers =new HashMap(); drivers.putAll(DataAccess.getStorageDriverLabels()); //Add an entry for the default (inherited from an ancestor or the system default) - drivers.put(getDefaultStorageDriverLabel(), "default"); + drivers.put(getDefaultStorageDriverLabel(), DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER); return drivers.entrySet(); } public String getDefaultStorageDriverLabel() { String storageDriverId = DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER; Dataverse parent = dataverse.getOwner(); + boolean fromAncestor=false; if(parent != null) { storageDriverId = parent.getEffectiveStorageDriverId(); - } - boolean fromAncestor=false; - if(!storageDriverId.equals(DataAccess.DEFAULT_STORAGE_DRIVER_IDENTIFIER)) { - fromAncestor = true; + //recurse dataverse chain to root and if any have a storagedriver set, fromAncestor is true + while(parent!=null) { + if(!parent.getStorageDriverId().equals(DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER)) { + fromAncestor=true; + break; + } + parent=parent.getOwner(); + } } String label = DataAccess.getStorageDriverLabelFor(storageDriverId); if(fromAncestor) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java index c5f358e8f71..1474104d379 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Admin.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Admin.java @@ -1665,7 +1665,7 @@ public Response getStorageDriver(@PathParam("alias") String alias) throws Wrappe } catch (WrappedResponse wr) { return wr.getResponse(); } - //Note that this returns what's set directly on this dataverse. If null, the user would have to recurse the chain of parents to find the effective storageDriver + //Note that this returns what's set directly on this dataverse. If null/DataAccess.UNDEFINED_STORAGE_DRIVER_IDENTIFIER, the user would have to recurse the chain of parents to find the effective storageDriver return ok(dataverse.getStorageDriverId()); } diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index ed4fc24742a..4955f404e40 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -41,9 +41,10 @@ public DataAccess() { }; - //Default is only for tests - public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id", "file"); + public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); + public static final String UNDEFINED_STORAGE_DRIVER_IDENTIFIER = "undefined"; //Used in dataverse.xhtml as a non-null selection option value (indicating a null driver/inheriting the default) + // The getStorageIO() methods initialize StorageIO objects for // datafiles that are already saved using one of the supported Dataverse // DataAccess IO drivers. diff --git a/src/main/webapp/dataverse.xhtml b/src/main/webapp/dataverse.xhtml index 8530b149001..06edc64f1a6 100644 --- a/src/main/webapp/dataverse.xhtml +++ b/src/main/webapp/dataverse.xhtml @@ -147,7 +147,7 @@ data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['dataverse.storage.title']}">
- + From 17936daf787bac9dfd0db4b40a0cf82ab5e9b5fb Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 12:30:07 -0500 Subject: [PATCH 106/157] using S3 tag for temp files --- .../iq/dataverse/dataaccess/S3AccessIO.java | 28 +++++++++++++++++++ .../dataverse/ingest/IngestServiceBean.java | 4 +++ src/main/webapp/resources/js/fileupload.js | 1 + 3 files changed, 33 insertions(+) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 9180eb502d2..bb4c6c9a078 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -7,10 +7,12 @@ import com.amazonaws.client.builder.AwsClientBuilder; import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.Headers; import com.amazonaws.services.s3.model.ObjectMetadata; import com.amazonaws.services.s3.model.PutObjectRequest; import com.amazonaws.services.s3.model.CopyObjectRequest; import com.amazonaws.services.s3.model.DeleteObjectRequest; +import com.amazonaws.services.s3.model.DeleteObjectTaggingRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion; import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest; @@ -851,6 +853,9 @@ public String generateTemporaryS3UploadUrl() throws IOException { GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key).withMethod(HttpMethod.PUT).withExpiration(expiration); + //Require user to add this header to indicate a temporary file + generatePresignedUrlRequest.putCustomRequestHeader(Headers.S3_TAGGING, "dv-state=temp"); + URL presignedUrl; try { presignedUrl = s3.generatePresignedUrl(generatePresignedUrlRequest); @@ -925,4 +930,27 @@ private void readSettings() { bucketName = System.getProperty("dataverse.files." + this.driverId + ".bucket-name"); } + + public void removeTempTag() throws IOException { + if (!(dvObject instanceof DataFile)) { + logger.warning("Attempt to remove tag from non-file DVObject id: " + dvObject.getId()); + throw new IOException("Attempt to remove temp tag from non-file S3 Object"); + } + try { + + key = getMainFileKey(); + DeleteObjectTaggingRequest deleteObjectTaggingRequest = new DeleteObjectTaggingRequest(bucketName, key); + //NOte - currently we only use one tag so delete is the fastest and cheapest way to get rid of that one tag + //Otherwise you have to get tags, remove the one you don't want and post new tags and get charged for the operations + s3.deleteObjectTagging(deleteObjectTaggingRequest); + } catch (SdkClientException sce) { + logger.severe("Unable to remove temp tag from : " + bucketName + " : " + key); + throw new IOException("Cannot remove temp tag from S3 object " + bucketName + " : " + key + " ("+sce.getMessage()+")"); + } catch (IOException e) { + logger.warning("Could not create key for S3 object." ); + e.printStackTrace(); + } + + } + } diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index c50257956f4..b9573b4aade 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -44,6 +44,7 @@ import edu.harvard.iq.dataverse.dataaccess.DataAccessOption; import edu.harvard.iq.dataverse.dataaccess.StorageIO; import edu.harvard.iq.dataverse.dataaccess.ImageThumbConverter; +import edu.harvard.iq.dataverse.dataaccess.S3AccessIO; import edu.harvard.iq.dataverse.dataaccess.TabularSubsetGenerator; import edu.harvard.iq.dataverse.datavariable.SummaryStatistic; import edu.harvard.iq.dataverse.datavariable.DataVariable; @@ -301,6 +302,9 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List)dataAccess).removeTempTag(); + } } catch (IOException ioex) { logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " (" + ioex.getMessage() + ")"); diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 83d42b9f417..b177fed7df7 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -51,6 +51,7 @@ function uploadFileDirectly(url, storageId) { $('.ui-fileupload-progress').append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); $.ajax({ url: url, + headers: {"x-amz-tagging":"dv-state=temp"}, type: 'PUT', data: file, cache: false, From 34c8ee2377428db1b89cfac80e07d649bd62dda3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 14:03:55 -0500 Subject: [PATCH 107/157] ignore tag issues on Minio --- .../edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 9 +++++++-- .../harvard/iq/dataverse/ingest/IngestServiceBean.java | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index bb4c6c9a078..7b6979ff1a7 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -944,8 +944,13 @@ public void removeTempTag() throws IOException { //Otherwise you have to get tags, remove the one you don't want and post new tags and get charged for the operations s3.deleteObjectTagging(deleteObjectTaggingRequest); } catch (SdkClientException sce) { - logger.severe("Unable to remove temp tag from : " + bucketName + " : " + key); - throw new IOException("Cannot remove temp tag from S3 object " + bucketName + " : " + key + " ("+sce.getMessage()+")"); + if(sce.getMessage().contains("Status Code: 501")) { + // In this case, it's likely that tags are not implemented at all (e.g. by Minio) so no tag was set either and it's just something to be aware of + logger.warning("Temp tag not deleted: Object tags not supported by storage: " + driverId); + } else { + // In this case, the assumption is that adding tags has worked, so not removing it is a problem that should be looked into. + logger.severe("Unable to remove temp tag from : " + bucketName + " : " + key); + } } catch (IOException e) { logger.warning("Could not create key for S3 object." ); e.printStackTrace(); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index b9573b4aade..c2a2a13c272 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -303,7 +303,7 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List)dataAccess).removeTempTag(); + ((S3AccessIO)dataAccess).removeTempTag(); } } catch (IOException ioex) { logger.warning("Failed to get file size, storage id " + dataFile.getStorageIdentifier() + " (" From 5c721ff24512bb2b8036f74d3ffd6b6b0e1a82f6 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 14:04:05 -0500 Subject: [PATCH 108/157] new progress color --- src/main/webapp/resources/css/structure.css | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/resources/css/structure.css b/src/main/webapp/resources/css/structure.css index 44ee913c385..e6fd0a110b6 100644 --- a/src/main/webapp/resources/css/structure.css +++ b/src/main/webapp/resources/css/structure.css @@ -979,5 +979,5 @@ progress::-webkit-progress-bar { } progress::-webkit-progress-value { - background-color:#0e90d2; + background-color:green; } From 2a78a68e9125fd28915bf42f088af9cb14364dd5 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 17:35:03 -0500 Subject: [PATCH 109/157] remove obsolete storage default string --- src/main/java/propertyFiles/Bundle.properties | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/java/propertyFiles/Bundle.properties b/src/main/java/propertyFiles/Bundle.properties index ae4dcab6a57..504f3bd3dec 100755 --- a/src/main/java/propertyFiles/Bundle.properties +++ b/src/main/java/propertyFiles/Bundle.properties @@ -693,7 +693,6 @@ dataverse.host.autocomplete.nomatches=No matches dataverse.identifier.title=Short name used for the URL of this dataverse. dataverse.affiliation.title=The organization with which this dataverse is affiliated. dataverse.storage.title=A storage service to be used for datasets in this dataverse. -dataverse.storage.usedefault=Use Default dataverse.category=Category dataverse.category.title=The type that most closely reflects this dataverse. dataverse.type.selectTab.top=Select one... From 66a48ee2cfa98454989751dca7d1a49e6f708169 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 19:07:48 -0500 Subject: [PATCH 110/157] remove unused methods and tests of them --- .../datasetutility/FileSizeChecker.java | 111 +++--------------- .../datasetutility/FileSizeCheckerTest.java | 67 ----------- 2 files changed, 19 insertions(+), 159 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileSizeChecker.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileSizeChecker.java index 8d24270c76c..06b3f467867 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileSizeChecker.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileSizeChecker.java @@ -6,9 +6,6 @@ package edu.harvard.iq.dataverse.datasetutility; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.SystemConfig; -import java.util.Collections; -import java.util.logging.Logger; /** * Convenience methods for checking max. file size @@ -16,94 +13,24 @@ */ public class FileSizeChecker { - private static final Logger logger = Logger.getLogger(FileSizeChecker.class.getCanonicalName()); + /* This method turns a number of bytes into a human readable version + */ + public static String bytesToHumanReadable(long v) { + return bytesToHumanReadable(v, 1); + } + + /* This method turns a number of bytes into a human readable version + * with figs decimal places + */ + public static String bytesToHumanReadable(long v, int figs) { + if (v < 1024) { + return v + " " + BundleUtil.getStringFromBundle("file.addreplace.error.byte_abrev"); + } + // 63 - because long has 63 binary digits + int trailingBin0s = (63 - Long.numberOfLeadingZeros(v))/10; + //String base = "%."+figs+"f %s"+ BundleUtil.getStringFromBundle("file.addreplace.error.byte_abrev"); + return String.format("%."+figs+"f %s"+ BundleUtil.getStringFromBundle("file.addreplace.error.byte_abrev"), (double)v / (1L << (trailingBin0s*10)), + " KMGTPE".charAt(trailingBin0s)); + } - SystemConfig systemConfig; - - /** - * constructor - */ - public FileSizeChecker(SystemConfig systemConfig){ - if (systemConfig == null){ - throw new NullPointerException("systemConfig cannot be null"); - } - this.systemConfig = systemConfig; - } - - public FileSizeResponse isAllowedFileSize(Long filesize){ - - if (filesize == null){ - throw new NullPointerException("filesize cannot be null"); - //return new FileSizeResponse(false, "The file size could not be found!"); - } - - Long maxFileSize = systemConfig.getMaxFileUploadSize(); - - // If no maxFileSize in the database, set it to unlimited! - // - if (maxFileSize == null){ - return new FileSizeResponse(true, - BundleUtil.getStringFromBundle("file.addreplace.file_size_ok") - ); - } - - // Good size! - // - if (filesize <= maxFileSize){ - return new FileSizeResponse(true, - BundleUtil.getStringFromBundle("file.addreplace.file_size_ok") - ); - } - - // Nope! Sorry! File is too big - // - String errMsg = BundleUtil.getStringFromBundle("file.addreplace.error.file_exceeds_limit", Collections.singletonList(bytesToHumanReadable(maxFileSize))); - - return new FileSizeResponse(false, errMsg); - - } - - /* This method turns a number of bytes into a human readable version - */ - public static String bytesToHumanReadable(long v) { - return bytesToHumanReadable(v, 1); - } - - /* This method turns a number of bytes into a human readable version - * with figs decimal places - */ - public static String bytesToHumanReadable(long v, int figs) { - if (v < 1024) { - return v + " " + BundleUtil.getStringFromBundle("file.addreplace.error.byte_abrev"); - } - // 63 - because long has 63 binary digits - int trailingBin0s = (63 - Long.numberOfLeadingZeros(v))/10; - //String base = "%."+figs+"f %s"+ BundleUtil.getStringFromBundle("file.addreplace.error.byte_abrev"); - return String.format("%."+figs+"f %s"+ BundleUtil.getStringFromBundle("file.addreplace.error.byte_abrev"), (double)v / (1L << (trailingBin0s*10)), - " KMGTPE".charAt(trailingBin0s)); - } - - /** - * Inner class that can also return an error message - */ - public class FileSizeResponse{ - - public boolean fileSizeOK; - public String userMsg; - - public FileSizeResponse(boolean isOk, String msg){ - - fileSizeOK = isOk; - userMsg = msg; - } - - public boolean isFileSizeOK(){ - return fileSizeOK; - } - - public String getUserMessage(){ - return userMsg; - } - - } // end inner class } diff --git a/src/test/java/edu/harvard/iq/dataverse/datasetutility/FileSizeCheckerTest.java b/src/test/java/edu/harvard/iq/dataverse/datasetutility/FileSizeCheckerTest.java index e562e3f9e0c..824dc6794fe 100644 --- a/src/test/java/edu/harvard/iq/dataverse/datasetutility/FileSizeCheckerTest.java +++ b/src/test/java/edu/harvard/iq/dataverse/datasetutility/FileSizeCheckerTest.java @@ -7,22 +7,13 @@ import static edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.bytesToHumanReadable; -import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker.FileSizeResponse; import edu.harvard.iq.dataverse.util.BundleUtil; -import edu.harvard.iq.dataverse.util.SystemConfig; - import java.util.ArrayList; import java.util.Arrays; import java.util.List; import org.junit.jupiter.api.Test; -import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.ValueSource; - import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; /** * @@ -44,62 +35,4 @@ public void testBytesToHumanReadable() { assertEquals(expAns, ans); assertEquals(expLongAns, longAns); } - - @Test - public void testIsAllowedFileSize_throwsOnNull() { - FileSizeChecker fileSizeChecker = new FileSizeChecker(new SystemConfig() { - @Override - public Long getMaxFileUploadSize() { - return 1000L; - } - }); - assertThrows(NullPointerException.class, () -> { - fileSizeChecker.isAllowedFileSize(null); - }); - } - - @ParameterizedTest - @ValueSource(longs = { 0L, 999L, 1000L }) - public void testIsAllowedFileSize_allowsSmallerOrEqualFileSize(Long fileSize) { - // initialize a system config and instantiate a file size checker - // override the max file upload side to allow for testing - FileSizeChecker fileSizeChecker = new FileSizeChecker(new SystemConfig() { - @Override - public Long getMaxFileUploadSize() { - return 1000L; - } - }); - FileSizeResponse response = fileSizeChecker.isAllowedFileSize(fileSize); - assertTrue(response.fileSizeOK); - } - - @ParameterizedTest - @ValueSource(longs = { 1001L, Long.MAX_VALUE }) - public void testIsAllowedFileSize_rejectsBiggerFileSize(Long fileSize) { - // initialize a system config and instantiate a file size checker - // override the max file upload side to allow for testing - FileSizeChecker fileSizeChecker = new FileSizeChecker(new SystemConfig() { - @Override - public Long getMaxFileUploadSize() { - return 1000L; - } - }); - FileSizeResponse response = fileSizeChecker.isAllowedFileSize(fileSize); - assertFalse(response.fileSizeOK); - } - - @ParameterizedTest - @ValueSource(longs = { 0L, 1000L, Long.MAX_VALUE }) - public void testIsAllowedFileSize_allowsOnUnboundedFileSize(Long fileSize) { - // initialize a system config and instantiate a file size checker - // ensure that a max filesize is not set - FileSizeChecker unboundedFileSizeChecker = new FileSizeChecker(new SystemConfig() { - @Override - public Long getMaxFileUploadSize() { - return null; - } - }); - FileSizeResponse response = unboundedFileSizeChecker.isAllowedFileSize(fileSize); - assertTrue(response.fileSizeOK); - } } From 560c97338a9a9e7d94fc7547221a1efebc19d7d3 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 3 Mar 2020 19:08:29 -0500 Subject: [PATCH 111/157] make max file size limit per store --- .../edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../iq/dataverse/EditDatafilesPage.java | 16 +++---- .../datadeposit/SwordConfigurationImpl.java | 7 ++- .../settings/SettingsServiceBean.java | 46 ++++++++++++++++++- .../harvard/iq/dataverse/util/FileUtil.java | 2 +- .../iq/dataverse/util/SystemConfig.java | 8 ++-- 6 files changed, 64 insertions(+), 17 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 690443d4def..67a09a9a62a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1774,7 +1774,7 @@ public void updateOwnerDataverse() { private String init(boolean initFull) { //System.out.println("_YE_OLDE_QUERY_COUNTER_"); // for debug purposes - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSize(); + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); setDataverseSiteUrl(systemConfig.getDataverseSiteUrl()); guestbookResponse = new GuestbookResponse(); diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 681c9c15dd9..971a62666f2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -450,10 +450,7 @@ public String initCreateMode(String modeToken, DatasetVersion version, List(); selectedFiles = selectedFileMetadatasList; + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); + this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); + logger.fine("done"); saveEnabled = true; @@ -473,9 +473,6 @@ public String init() { newFiles = new ArrayList<>(); uploadedFiles = new ArrayList<>(); - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSize(); - this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); - if (dataset.getId() != null){ // Set Working Version and Dataset by Datasaet Id and Version //retrieveDatasetVersionResponse = datasetVersionService.retrieveDatasetVersionById(dataset.getId(), null); @@ -490,7 +487,10 @@ public String init() { // that the dataset id is mandatory... But 404 will do for now. return permissionsWrapper.notFound(); } - + + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); + this.multipleUploadFilesLimit = systemConfig.getMultipleUploadFilesLimit(); + workingVersion = dataset.getEditVersion(); clone = workingVersion.cloneDatasetVersion(); if (workingVersion == null || !workingVersion.isDraft()) { diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordConfigurationImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordConfigurationImpl.java index 4eb6e77fe21..ce5f9415fcc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordConfigurationImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/SwordConfigurationImpl.java @@ -124,8 +124,11 @@ public String getTempDirectory() { public int getMaxUploadSize() { int unlimited = -1; - - Long maxUploadInBytes = systemConfig.getMaxFileUploadSize(); + /* It doesn't look like we can determine which store will be used here, so we'll go with the default + * (It looks like the collection or study involved is available where this method is called, but the SwordConfiguration.getMaxUploadSize() + * doesn't allow a parameter) + */ + Long maxUploadInBytes = systemConfig.getMaxFileUploadSizeForStore("default"); if (maxUploadInBytes == null){ // (a) No setting, return unlimited diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 473bea561b4..75b604eb440 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -1,9 +1,13 @@ package edu.harvard.iq.dataverse.settings; +import edu.harvard.iq.dataverse.Dataset; import edu.harvard.iq.dataverse.actionlogging.ActionLogRecord; import edu.harvard.iq.dataverse.actionlogging.ActionLogServiceBean; import edu.harvard.iq.dataverse.api.ApiBlockingFilter; +import edu.harvard.iq.dataverse.engine.command.DataverseRequest; import edu.harvard.iq.dataverse.util.StringUtil; + +import java.io.StringReader; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -12,6 +16,8 @@ import javax.ejb.EJB; import javax.ejb.Stateless; import javax.inject.Named; +import javax.json.Json; +import javax.json.JsonObject; import javax.persistence.EntityManager; import javax.persistence.PersistenceContext; @@ -162,7 +168,7 @@ public enum Key { /** Enable full-text indexing in solr up to max file size */ SolrFullTextIndexing, //true or false (default) SolrMaxFileSizeForFullTextIndexing, //long - size in bytes (default unset/no limit) - /** Key for limiting the number of bytes uploaded via the Data Deposit API, UI (web site and . */ + /** Default Key for limiting the number of bytes uploaded via the Data Deposit API, UI (web site and . */ MaxFileUploadSizeInBytes, /** Key for if ScrubMigrationData is enabled or disabled. */ ScrubMigrationData, @@ -477,6 +483,44 @@ public Long getValueForKeyAsLong(Key key){ } + /** + * Attempt to convert a value in a compound key to a long + * - Applicable for keys such as MaxFileUploadSizeInBytes after multistore capabilities were added in ~v4.20 + * backward compatible with a single value. For multi values, the key's value must be an object with param:value pairs. + * A "default":value pair is allowed and will be returned for any param that doesn't have a defined value. + * + * On failure (key not found or string not convertible to a long), returns null + * @param key + * @return + */ + public Long getValueForCompoundKeyAsLong(Key key, String param){ + + String val = this.getValueForKey(key); + + if (val == null){ + return null; + } + + try { + return Long.parseLong(val); + } catch (NumberFormatException ex) { + try ( StringReader rdr = new StringReader(val) ) { + JsonObject settings = Json.createReader(rdr).readObject(); + if(settings.containsKey(param)) { + return Long.parseLong(settings.getString(param)); + } else if(settings.containsKey("default")) { + return Long.parseLong(settings.getString("default")); + } else { + return null; + } + + } catch (Exception e) { + logger.log(Level.WARNING, "Incorrect setting. Could not convert \"{0}\" from setting {1} to long.", new Object[]{val, key.toString()}); + return null; + } + } + + } /** * Return the value stored, or the default value, in case no setting by that diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index 820a7037605..f272e3ff884 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -714,7 +714,7 @@ public static List createDataFiles(DatasetVersion version, InputStream // save the file, in the temporary location for now: Path tempFile = null; - Long fileSizeLimit = systemConfig.getMaxFileUploadSize(); + Long fileSizeLimit = systemConfig.getMaxFileUploadSizeForStore(version.getDataset().getOwner().getEffectiveStorageDriverId()); String finalType = null; if (newStorageIdentifier == null) { if (getFilesTempDirectory() != null) { diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 8d0cb276a93..161493fa13f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -539,12 +539,12 @@ public boolean isFilesOnDatasetPageFromSolr() { return settingsService.isTrueForKey(SettingsServiceBean.Key.FilesOnDatasetPageFromSolr, safeDefaultIfKeyNotFound); } - public Long getMaxFileUploadSize(){ - return settingsService.getValueForKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes); + public Long getMaxFileUploadSizeForStore(String driverId){ + return settingsService.getValueForCompoundKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes, driverId); } - public String getHumanMaxFileUploadSize(){ - return bytesToHumanReadable(getMaxFileUploadSize()); + public String getHumanMaxFileUploadSizeForStore(String driverId){ + return bytesToHumanReadable(getMaxFileUploadSizeForStore(driverId)); } public Integer getSearchHighlightFragmentSize() { From 84f57a31196f04c7a72462aa6403814cd31ee53e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 4 Mar 2020 10:37:43 -0500 Subject: [PATCH 112/157] get max file size after dataset is initialized --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 67a09a9a62a..ae89602e045 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1774,7 +1774,6 @@ public void updateOwnerDataverse() { private String init(boolean initFull) { //System.out.println("_YE_OLDE_QUERY_COUNTER_"); // for debug purposes - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); setDataverseSiteUrl(systemConfig.getDataverseSiteUrl()); guestbookResponse = new GuestbookResponse(); @@ -1821,7 +1820,10 @@ private String init(boolean initFull) { // Set Working Version and Dataset by DatasaetVersion Id //retrieveDatasetVersionResponse = datasetVersionService.retrieveDatasetVersionByVersionId(versionId); - } + } + + this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); + if (retrieveDatasetVersionResponse == null) { return permissionsWrapper.notFound(); From 500094ff15df0499fc4b08c70abc10dff8e2a539 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 4 Mar 2020 14:10:41 -0500 Subject: [PATCH 113/157] remove systemconfig default methods --- src/main/java/edu/harvard/iq/dataverse/DatasetPage.java | 1 - .../java/edu/harvard/iq/dataverse/EditDatafilesPage.java | 5 +++++ .../harvard/iq/dataverse/settings/SettingsServiceBean.java | 2 +- .../java/edu/harvard/iq/dataverse/util/SystemConfig.java | 4 ---- src/main/webapp/editFilesFragment.xhtml | 4 ++-- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index ae89602e045..b13200b0dde 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -1821,7 +1821,6 @@ private String init(boolean initFull) { //retrieveDatasetVersionResponse = datasetVersionService.retrieveDatasetVersionByVersionId(versionId); } - this.maxFileUploadSizeInBytes = systemConfig.getMaxFileUploadSizeForStore(dataset.getOwner().getEffectiveStorageDriverId()); diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 971a62666f2..71b74ddc7ae 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -7,6 +7,7 @@ import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; import edu.harvard.iq.dataverse.branding.BrandingUtil; import edu.harvard.iq.dataverse.datasetutility.AddReplaceFileHelper; +import edu.harvard.iq.dataverse.datasetutility.FileSizeChecker; import edu.harvard.iq.dataverse.datasetutility.FileReplaceException; import edu.harvard.iq.dataverse.datasetutility.FileReplacePageHelper; import edu.harvard.iq.dataverse.dataaccess.DataAccess; @@ -313,6 +314,10 @@ public Long getMaxFileUploadSizeInBytes() { return this.maxFileUploadSizeInBytes; } + public String getHumanMaxFileUploadSizeInBytes() { + return FileSizeChecker.bytesToHumanReadable(this.maxFileUploadSizeInBytes); + } + public boolean isUnlimitedUploadFileSize() { return this.maxFileUploadSizeInBytes == null; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 75b604eb440..ece5eff2843 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -515,7 +515,7 @@ public Long getValueForCompoundKeyAsLong(Key key, String param){ } } catch (Exception e) { - logger.log(Level.WARNING, "Incorrect setting. Could not convert \"{0}\" from setting {1} to long.", new Object[]{val, key.toString()}); + logger.log(Level.WARNING, "Incorrect setting. Could not convert \"{0}\" from setting {1} to long: {2}", new Object[]{val, key.toString(), e.getMessage()}); return null; } } diff --git a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java index 161493fa13f..aefb01992f4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/SystemConfig.java @@ -543,10 +543,6 @@ public Long getMaxFileUploadSizeForStore(String driverId){ return settingsService.getValueForCompoundKeyAsLong(SettingsServiceBean.Key.MaxFileUploadSizeInBytes, driverId); } - public String getHumanMaxFileUploadSizeForStore(String driverId){ - return bytesToHumanReadable(getMaxFileUploadSizeForStore(driverId)); - } - public Integer getSearchHighlightFragmentSize() { String fragSize = settingsService.getValueForKey(SettingsServiceBean.Key.SearchHighlightFragmentSize); if (fragSize != null) { diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 5f3b14d92b5..57d760f6a56 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -73,8 +73,8 @@ - + rendered="#{!EditDatafilesPage.isUnlimitedUploadFileSize()}"> +

From 91136126c9cbcbbe4b25a68803c87103a1694d9e Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 4 Mar 2020 14:29:18 -0500 Subject: [PATCH 114/157] restore test fix --- .../java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index 4955f404e40..db87b1751c6 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -41,8 +41,8 @@ public DataAccess() { }; - - public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id"); + //Default to "file" is for tests only + public static final String DEFAULT_STORAGE_DRIVER_IDENTIFIER = System.getProperty("dataverse.files.storage-driver-id", "file"); public static final String UNDEFINED_STORAGE_DRIVER_IDENTIFIER = "undefined"; //Used in dataverse.xhtml as a non-null selection option value (indicating a null driver/inheriting the default) // The getStorageIO() methods initialize StorageIO objects for From d3c2539f2a699725bbc8bbe546dd9700bd0a4311 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 4 Mar 2020 14:49:37 -0500 Subject: [PATCH 115/157] document per store file upload size limits --- doc/sphinx-guides/source/installation/config.rst | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 7920661378b..ec82b035a32 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -244,6 +244,8 @@ If you wish to change which store is used by default, you'll need to delete the ./asadmin $ASADMIN_OPTS delete-jvm-options "-Ddataverse.files.storage-driver-id=file" ./asadmin $ASADMIN_OPTS create-jvm-options "-Ddataverse.files.storage-driver-id=" + + It is also possible to set maximum file upload size limits per store. See the :ref:`:MaxFileUploadSizeInBytes` setting below. File Storage ++++++++++++ @@ -1476,7 +1478,14 @@ Alongside the ``:StatusMessageHeader`` you need to add StatusMessageText for the :MaxFileUploadSizeInBytes +++++++++++++++++++++++++ -Set `MaxFileUploadSizeInBytes` to "2147483648", for example, to limit the size of files uploaded to 2 GB. +This setting controls the maximum size of uploaded files. +- To have one limit for all stores, set `MaxFileUploadSizeInBytes` to "2147483648", for example, to limit the size of files uploaded to 2 GB: + +``curl -X PUT -d 2147483648 http://localhost:8080/api/admin/settings/:MaxFileUploadSizeInBytes`` + +- To have limits per store with an optional default, use a serialized json object for the value of `MaxFileUploadSizeInBytes` with an entry per store, as in the following exmaple, which maintains a 2 GB default and adds higher limits for stores with ids "fileOne" and "s3". + +``curl -X PUT -d '{"default":"2147483648","fileOne":"4000000000","s3":"8000000000"}' http://localhost:8080/api/admin/settings/:MaxFileUploadSizeInBytes`` Notes: @@ -1486,7 +1495,7 @@ Notes: - For larger file upload sizes, you may need to configure your reverse proxy timeout. If using apache2 (httpd) with Shibboleth, add a timeout to the ProxyPass defined in etc/httpd/conf.d/ssl.conf (which is described in the :doc:`/installation/shibboleth` setup). -``curl -X PUT -d 2147483648 http://localhost:8080/api/admin/settings/:MaxFileUploadSizeInBytes`` + :ZipDownloadLimit +++++++++++++++++ From abbebd8a3167dc6b99350c667871ae626846dd82 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 4 Mar 2020 15:06:37 -0500 Subject: [PATCH 116/157] doc of new features --- doc/sphinx-guides/source/developers/big-data-support.rst | 4 ++++ doc/sphinx-guides/source/installation/config.rst | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index 8f1103cf720..a64dbb07ea1 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -23,6 +23,8 @@ To configure these options, an administrator must set two JVM options for the Da With multiple stores configured, it is possible to configure one S3 store with direct upload and/or download to support large files (in general or for specific dataverses) while configuring only direct download, or no direct access for another store. +It is also possible to set file upload size limits per store. See the :MaxFileUploadSizeInBytes setting described in the :doc:`/installation/config` guide. + At present, one potential drawback for direct-upload is that files are only partially 'ingested', tabular and FITS files are processed, but zip files are not unzipped, and the file contents are not inspected to evaluate their mimetype. This could be appropriate for large files, or it may be useful to completely turn off ingest processing for performance reasons (ingest processing requires a copy of the file to be retrieved by Dataverse from the S3 store). A store using direct upload can be configured to disable all ingest processing for files above a given size limit: ``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="`` @@ -30,6 +32,8 @@ At present, one potential drawback for direct-upload is that files are only part One additional step that is required to enable direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. +Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-status":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in Dataverse. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. + Data Capture Module (DCM) ------------------------- diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index ec82b035a32..bb2f5702899 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1483,7 +1483,7 @@ This setting controls the maximum size of uploaded files. ``curl -X PUT -d 2147483648 http://localhost:8080/api/admin/settings/:MaxFileUploadSizeInBytes`` -- To have limits per store with an optional default, use a serialized json object for the value of `MaxFileUploadSizeInBytes` with an entry per store, as in the following exmaple, which maintains a 2 GB default and adds higher limits for stores with ids "fileOne" and "s3". +- To have limits per store with an optional default, use a serialized json object for the value of `MaxFileUploadSizeInBytes` with an entry per store, as in the following example, which maintains a 2 GB default and adds higher limits for stores with ids "fileOne" and "s3". ``curl -X PUT -d '{"default":"2147483648","fileOne":"4000000000","s3":"8000000000"}' http://localhost:8080/api/admin/settings/:MaxFileUploadSizeInBytes`` From 5cd8744bb30e64d471a125a3fc48d644d853728d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 5 Mar 2020 15:40:40 -0500 Subject: [PATCH 117/157] avoid resetting mimetype for normal upload --- src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java index f272e3ff884..a4370c7b38f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/FileUtil.java @@ -1084,10 +1084,7 @@ public static List createDataFiles(DatasetVersion version, InputStream } return null; - } else { - // ToDo what subset of checks from determineFileType makes sense? - finalType = suppliedContentType; - } + } } else { //Remote file, trust supplier finalType = suppliedContentType; From bce3c33ec3f40a89bb3d7529b4bc97e57a2c1bb2 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Fri, 6 Mar 2020 14:51:35 +0100 Subject: [PATCH 118/157] Issue #6514: Implement affiliation reading from Shibboleth attribute. --- .../source/installation/config.rst | 17 +++++++++++++++++ .../java/edu/harvard/iq/dataverse/Shib.java | 11 ++++++++++- .../dataverse/settings/SettingsServiceBean.java | 8 +++++++- 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index bfa66b97eb1..154f0bbff3e 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1777,6 +1777,23 @@ You can set the value of "#THIS PAGE#" to the URL of your Dataverse homepage, or ``curl -X PUT -d true http://localhost:8080/api/admin/settings/:ShibPassiveLoginEnabled`` +:ShibAffiliationAttribute ++++++++++++++++++++++++++ + +Shibboleth affiliation attribute which holds information about the affiliation of the user (e.g. "ou"). In case of Shibboleth affiliation string is read from the DiscoFeed at each login. ``:ShibAffiliationAttribute`` is a name of a Shibboleth attribute, which takes place in Shibboleth header, and Dataverse will read the affiliation string from that. If this value is not set or empty, Dataverse uses the DiscoFeed. + +To set ``:ShibAffiliationAttribute``: + +``curl -X PUT -d "ou" http://localhost:8080/api/admin/settings/:ShibAffiliationAttribute`` + +To delete ``:ShibAffiliationAttribute``: + +``curl -X DELETE http://localhost:8080/api/admin/settings/:ShibAffiliationAttribute`` + +To check the current value of ``:ShibAffiliationAttribute``: + +``curl -X GET http://localhost:8080/api/admin/settings/:ShibAffiliationAttribute`` + .. _:ComputeBaseUrl: :ComputeBaseUrl diff --git a/src/main/java/edu/harvard/iq/dataverse/Shib.java b/src/main/java/edu/harvard/iq/dataverse/Shib.java index 8af9a22d783..d5ff8d88de5 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Shib.java +++ b/src/main/java/edu/harvard/iq/dataverse/Shib.java @@ -11,9 +11,12 @@ import edu.harvard.iq.dataverse.authorization.providers.shib.ShibUserNameFields; import edu.harvard.iq.dataverse.authorization.providers.shib.ShibUtil; import edu.harvard.iq.dataverse.authorization.users.AuthenticatedUser; +import edu.harvard.iq.dataverse.settings.SettingsServiceBean; import edu.harvard.iq.dataverse.util.BundleUtil; import edu.harvard.iq.dataverse.util.JsfHelper; import edu.harvard.iq.dataverse.util.SystemConfig; +import org.apache.commons.lang.StringUtils; + import java.io.IOException; import java.sql.Timestamp; import java.util.ArrayList; @@ -50,6 +53,8 @@ public class Shib implements java.io.Serializable { GroupServiceBean groupService; @EJB UserNotificationServiceBean userNotificationService; + @EJB + SettingsServiceBean settingsService; HttpServletRequest request; @@ -205,7 +210,11 @@ public void init() { internalUserIdentifer = ShibUtil.generateFriendlyLookingUserIdentifer(usernameAssertion, emailAddress); logger.fine("friendly looking identifer (backend will enforce uniqueness):" + internalUserIdentifer); - String affiliation = shibService.getAffiliation(shibIdp, shibService.getDevShibAccountType()); + String shibAffiliationAttribute = settingsService.getValueForKey(SettingsServiceBean.Key.ShibAffiliationAttribute); + String affiliation = (StringUtils.isNotBlank(shibAffiliationAttribute)) + ? getValueFromAssertion(shibAffiliationAttribute) + : shibService.getAffiliation(shibIdp, shibService.getDevShibAccountType()); + if (affiliation != null) { affiliationToDisplayAtConfirmation = affiliation; friendlyNameForInstitution = affiliation; diff --git a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java index 473bea561b4..165944acfdb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/settings/SettingsServiceBean.java @@ -411,7 +411,13 @@ Whether Harvesting (OAI) service is enabled * Lifespan, in minutes, of a login user session  * (both DataverseSession and the underlying HttpSession) */ - LoginSessionTimeout; + LoginSessionTimeout, + + /** + * Shibboleth affiliation attribute which holds information about the affiliation of the user (e.g. ou) + */ + ShibAffiliationAttribute + ; @Override public String toString() { From bd3f1d6803c36115976fcade9fb4d3a1bd1cd1b1 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Fri, 6 Mar 2020 15:35:11 +0100 Subject: [PATCH 119/157] #6514: extend documentation. --- .../source/installation/config.rst | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 154f0bbff3e..95cedf8ca32 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1782,6 +1782,36 @@ You can set the value of "#THIS PAGE#" to the URL of your Dataverse homepage, or Shibboleth affiliation attribute which holds information about the affiliation of the user (e.g. "ou"). In case of Shibboleth affiliation string is read from the DiscoFeed at each login. ``:ShibAffiliationAttribute`` is a name of a Shibboleth attribute, which takes place in Shibboleth header, and Dataverse will read the affiliation string from that. If this value is not set or empty, Dataverse uses the DiscoFeed. +If the attribute is not yet set for the Shibboleth, please consult the Shibboleth administrators how o set it. Typically it requires changing of `/etc/shibboleth/attribute-map.xml` file by adding an attribute request, e.g. + +``` + + + +``` + +In order to take place the change, you should restart Shibboleth and Apache2 services: + +``` +sudo service shibd restart +sudo service apache2 restart +``` + +To check if the attribute is sent, you should log in again to Dataverse and check Shibboleth's transaction log. You should see something like this: + +``` +INFO Shibboleth-TRANSACTION [25]: Cached the following attributes with session (ID: _9d1f34c0733b61c0feb0ca7596ef43b2) for (applicationId: default) { +INFO Shibboleth-TRANSACTION [25]: givenName (1 values) +INFO Shibboleth-TRANSACTION [25]: ou (1 values) +INFO Shibboleth-TRANSACTION [25]: sn (1 values) +INFO Shibboleth-TRANSACTION [25]: eppn (1 values) +INFO Shibboleth-TRANSACTION [25]: mail (1 values) +INFO Shibboleth-TRANSACTION [25]: displayName (1 values) +INFO Shibboleth-TRANSACTION [25]: } +``` + +If you see the attribue you requested in this list, you can set the attribute in Dataverse. + To set ``:ShibAffiliationAttribute``: ``curl -X PUT -d "ou" http://localhost:8080/api/admin/settings/:ShibAffiliationAttribute`` From b67125ba2282f58db0a238e6b824ae09f90f8f5f Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Fri, 6 Mar 2020 10:10:30 -0500 Subject: [PATCH 120/157] adding release note file, some basic doc updates --- doc/release-notes/6514-shib-updates | 0 doc/sphinx-guides/source/installation/config.rst | 6 +++--- 2 files changed, 3 insertions(+), 3 deletions(-) create mode 100644 doc/release-notes/6514-shib-updates diff --git a/doc/release-notes/6514-shib-updates b/doc/release-notes/6514-shib-updates new file mode 100644 index 00000000000..e69de29bb2d diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 95cedf8ca32..9fc2cf3cd11 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1780,9 +1780,9 @@ You can set the value of "#THIS PAGE#" to the URL of your Dataverse homepage, or :ShibAffiliationAttribute +++++++++++++++++++++++++ -Shibboleth affiliation attribute which holds information about the affiliation of the user (e.g. "ou"). In case of Shibboleth affiliation string is read from the DiscoFeed at each login. ``:ShibAffiliationAttribute`` is a name of a Shibboleth attribute, which takes place in Shibboleth header, and Dataverse will read the affiliation string from that. If this value is not set or empty, Dataverse uses the DiscoFeed. +The Shibboleth affiliation attribute holds information about the affiliation of the user (e.g. "ou"). The Shibboleth affiliation string is read from the DiscoFeed at each login. ``:ShibAffiliationAttribute`` is a name of a Shibboleth attribute, which takes place in Shibboleth header, and Dataverse will read the affiliation string from that. If this value is not set or empty, Dataverse uses the DiscoFeed. -If the attribute is not yet set for the Shibboleth, please consult the Shibboleth administrators how o set it. Typically it requires changing of `/etc/shibboleth/attribute-map.xml` file by adding an attribute request, e.g. +If the attribute is not yet set for the Shibboleth, please consult the Shibboleth administrators how to set it. Typically it requires changing of the `/etc/shibboleth/attribute-map.xml` file by adding an attribute request, e.g. ``` @@ -1790,7 +1790,7 @@ If the attribute is not yet set for the Shibboleth, please consult the Shibbolet ``` -In order to take place the change, you should restart Shibboleth and Apache2 services: +In order to implement the change, you should restart Shibboleth and Apache2 services: ``` sudo service shibd restart From d600593d6263c090f80e31ced576e0454febbac9 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Fri, 6 Mar 2020 10:22:02 -0500 Subject: [PATCH 121/157] Update 6489-release-notes.md --- doc/release-notes/6489-release-notes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/release-notes/6489-release-notes.md b/doc/release-notes/6489-release-notes.md index 3c21cd52bdb..5f38bd61d55 100644 --- a/doc/release-notes/6489-release-notes.md +++ b/doc/release-notes/6489-release-notes.md @@ -10,8 +10,8 @@ Direct upload to S3 is enabled per store by one new jvm option: ./asadmin create-jvm-options "\-Ddataverse.files..upload-redirect=true" -The existing :MaxFileUploadSizeInBytes property and dataverse.files..url-expiration-minutes jvm option for the same store also apply to direct upload. +The existing :MaxFileUploadSizeInBytes property and "dataverse.files..url-expiration-minutes" jvm option for the same store also apply to direct upload. Direct upload via the Dataverse web interface is transparent to the user and handled automatically by the browser. Some minor differences in file upload exist: directly uploaded files are not unzipped and Dataverse does not scan their content to help in assigning a MIME type. Ingest of tabular files and metadata extraction from FITS files will occur, but can be turned off for files above a specified size limit through the new dataverse.files..ingestsizelimit jvm option. -API calls to support direct upload also exist, and, if direct upload is enabled for a store in Dataverse, the latest DVUploader (v1.0.8) provides a'-directupload' flag that enables its use. \ No newline at end of file +API calls to support direct upload also exist, and, if direct upload is enabled for a store in Dataverse, the latest DVUploader (v1.0.8) provides a'-directupload' flag that enables its use. From 0f4b5f0266f9d46f395387c5617ddadb8bfc6029 Mon Sep 17 00:00:00 2001 From: Kevin Condon Date: Fri, 6 Mar 2020 10:27:06 -0500 Subject: [PATCH 122/157] Update 6489-release-notes.md --- doc/release-notes/6489-release-notes.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6489-release-notes.md b/doc/release-notes/6489-release-notes.md index 5f38bd61d55..b28174c9a74 100644 --- a/doc/release-notes/6489-release-notes.md +++ b/doc/release-notes/6489-release-notes.md @@ -10,7 +10,7 @@ Direct upload to S3 is enabled per store by one new jvm option: ./asadmin create-jvm-options "\-Ddataverse.files..upload-redirect=true" -The existing :MaxFileUploadSizeInBytes property and "dataverse.files..url-expiration-minutes" jvm option for the same store also apply to direct upload. +The existing :MaxFileUploadSizeInBytes property and ```dataverse.files..url-expiration-minutes``` jvm option for the same store also apply to direct upload. Direct upload via the Dataverse web interface is transparent to the user and handled automatically by the browser. Some minor differences in file upload exist: directly uploaded files are not unzipped and Dataverse does not scan their content to help in assigning a MIME type. Ingest of tabular files and metadata extraction from FITS files will occur, but can be turned off for files above a specified size limit through the new dataverse.files..ingestsizelimit jvm option. From e235d8c9854e64fdb44ebe5606f5c1ae90a4616b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 6 Mar 2020 14:55:32 -0500 Subject: [PATCH 123/157] initialize storageDriver to null affects root dataverse also checking isBlank on storagedriver rather than ==null as backward compatibility for devs. --- src/main/java/edu/harvard/iq/dataverse/Dataverse.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java index 0eea108c461..75dbb39e2ca 100644 --- a/src/main/java/edu/harvard/iq/dataverse/Dataverse.java +++ b/src/main/java/edu/harvard/iq/dataverse/Dataverse.java @@ -32,6 +32,8 @@ import javax.validation.constraints.NotNull; import javax.validation.constraints.Pattern; import javax.validation.constraints.Size; + +import org.apache.commons.lang.StringUtils; import org.hibernate.validator.constraints.NotBlank; import org.hibernate.validator.constraints.NotEmpty; @@ -149,7 +151,7 @@ public String getIndexableCategoryName() { private String affiliation; - private String storageDriver=""; + private String storageDriver=null; // Note: We can't have "Remove" here, as there are role assignments that refer // to this role. So, adding it would mean violating a forign key contstraint. @@ -762,7 +764,7 @@ public boolean isAncestorOf( DvObject other ) { public String getEffectiveStorageDriverId() { String id = storageDriver; - if(id == null) { + if(StringUtils.isBlank(id)) { if(this.getOwner() != null) { id = this.getOwner().getEffectiveStorageDriverId(); } else { From a8c8e008c135c5d5a5d5aaf2fbec7d94e8416709 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 6 Mar 2020 14:56:50 -0500 Subject: [PATCH 124/157] set alreadyExists(DVObject) false to allow direct upload during create mode with directupload, this will get called, and true would cause a failure --- .../dataverse/pidproviders/FakePidProviderServiceBean.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java index ce9e281e986..eb313631077 100644 --- a/src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/pidproviders/FakePidProviderServiceBean.java @@ -15,7 +15,11 @@ public class FakePidProviderServiceBean extends AbstractGlobalIdServiceBean { @Override public boolean alreadyExists(DvObject dvo) throws Exception { - return true; + /* Direct upload creates an identifier prior to calling the CreateNewDatasetCommand - if this is true, that call fails. + * In that case, the local test (DatasetServiceBean.isIdentifierLocallyUnique()) correctly returns false since it tests the database. + * This provider could do the same check or use some other method to test alreadyExists(DvObject) =true failures. (no tests found currently) + */ + return false; } @Override From 5584adcbd672ffc4e04cf2487329d446056b170d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 6 Mar 2020 15:11:17 -0500 Subject: [PATCH 125/157] remove debug logging --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 7b6979ff1a7..96ce92754a4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -866,9 +866,7 @@ public String generateTemporaryS3UploadUrl() throws IOException { String urlString = null; if (presignedUrl != null) { String endpoint = System.getProperty("dataverse.files." + driverId + ".custom-endpoint-url"); - logger.info("endpoint: " + endpoint); String proxy = System.getProperty("dataverse.files." + driverId + ".proxy-url"); - logger.info("proxy: " + proxy); if(proxy!=null) { urlString = presignedUrl.toString().replace(endpoint, proxy); } else { From cf237af1a2a0d526f32ccbdfb16a552a123a378c Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 9 Mar 2020 21:44:50 -0400 Subject: [PATCH 126/157] doc edits --- doc/sphinx-guides/source/installation/config.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/installation/config.rst b/doc/sphinx-guides/source/installation/config.rst index 9fc2cf3cd11..6bf03cef215 100644 --- a/doc/sphinx-guides/source/installation/config.rst +++ b/doc/sphinx-guides/source/installation/config.rst @@ -1780,9 +1780,9 @@ You can set the value of "#THIS PAGE#" to the URL of your Dataverse homepage, or :ShibAffiliationAttribute +++++++++++++++++++++++++ -The Shibboleth affiliation attribute holds information about the affiliation of the user (e.g. "ou"). The Shibboleth affiliation string is read from the DiscoFeed at each login. ``:ShibAffiliationAttribute`` is a name of a Shibboleth attribute, which takes place in Shibboleth header, and Dataverse will read the affiliation string from that. If this value is not set or empty, Dataverse uses the DiscoFeed. +The Shibboleth affiliation attribute holds information about the affiliation of the user (e.g. "OU") and is read from the DiscoFeed at each login. ``:ShibAffiliationAttribute`` is a name of a Shibboleth attribute in the Shibboleth header which Dataverse will read from instead of DiscoFeed. If this value is not set or empty, Dataverse uses the DiscoFeed. -If the attribute is not yet set for the Shibboleth, please consult the Shibboleth administrators how to set it. Typically it requires changing of the `/etc/shibboleth/attribute-map.xml` file by adding an attribute request, e.g. +If the attribute is not yet set for the Shibboleth, please consult the Shibboleth Administrators at your institution. Typically it requires changing of the `/etc/shibboleth/attribute-map.xml` file by adding an attribute request, e.g. ``` From c2133b10bcf9919e4c2557a87171f597891e2c7d Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Mon, 9 Mar 2020 21:45:48 -0400 Subject: [PATCH 127/157] add release note --- doc/release-notes/6514-shib-updates | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release-notes/6514-shib-updates b/doc/release-notes/6514-shib-updates index e69de29bb2d..6935fd1fc99 100644 --- a/doc/release-notes/6514-shib-updates +++ b/doc/release-notes/6514-shib-updates @@ -0,0 +1 @@ +New JVM option :ShibAffiliationAttribute \ No newline at end of file From 5a27cdf4d599fb1ad08b5ed1ab537dc54d430df1 Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 10 Mar 2020 09:54:15 -0400 Subject: [PATCH 128/157] changing JVM option to DB option --- doc/release-notes/6514-shib-updates | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/release-notes/6514-shib-updates b/doc/release-notes/6514-shib-updates index 6935fd1fc99..561358c6b8d 100644 --- a/doc/release-notes/6514-shib-updates +++ b/doc/release-notes/6514-shib-updates @@ -1 +1 @@ -New JVM option :ShibAffiliationAttribute \ No newline at end of file +New DB option :ShibAffiliationAttribute \ No newline at end of file From fdbf8db07d4cb5e381d253e7706f9d29b479662a Mon Sep 17 00:00:00 2001 From: Oliver Bertuch Date: Tue, 10 Mar 2020 15:29:51 +0100 Subject: [PATCH 129/157] Make BundleUtil.getDefaultLocale() respect sane system defaults. #6734 --- .../harvard/iq/dataverse/util/BundleUtil.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java index 98a2d84aee6..5ceff6b35ae 100644 --- a/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java +++ b/src/main/java/edu/harvard/iq/dataverse/util/BundleUtil.java @@ -159,19 +159,20 @@ public static String getStringFromDefaultPropertyFile(String key, String propert } return getStringFromBundleNoMissingCheck(key, null, bundle); } - + + /** + * Return JVM default locale. + * + * For now, this simply forwards default system behaviour. + * That means on JDK8 the system property user.language will be set on startup + * from environment variables like LANG or via Maven arguments (which is important for testing). + * (See also pom.xml for an example how we pinpoint this for reproducible tests!) + * (You should also be aware that good IDEs are honoring settings from pom.xml.) + * + * Nonetheless, someday we might want to have more influence on how this is determined, thus this wrapper. + * @return Dataverse default locale + */ public static Locale getDefaultLocale() { - String localeEnvVar = System.getenv().get("LANG"); - if (localeEnvVar != null) { - if (localeEnvVar.indexOf('.') > 0) { - localeEnvVar = localeEnvVar.substring(0, localeEnvVar.indexOf('.')); - } - if (!"en_US".equals(localeEnvVar)) { - logger.fine("BundleUtil: LOCALE code from the environmental variable is "+localeEnvVar); - return new Locale(localeEnvVar); - } - } - - return new Locale("en"); + return Locale.getDefault(); } } From 770194d3d0701742856bd86bcb287f50c1e372ba Mon Sep 17 00:00:00 2001 From: Danny Brooke Date: Tue, 10 Mar 2020 14:32:28 -0400 Subject: [PATCH 130/157] adding release note --- doc/release-notes/6650-export-import-mismatch | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 doc/release-notes/6650-export-import-mismatch diff --git a/doc/release-notes/6650-export-import-mismatch b/doc/release-notes/6650-export-import-mismatch new file mode 100644 index 00000000000..0ab2999a603 --- /dev/null +++ b/doc/release-notes/6650-export-import-mismatch @@ -0,0 +1,3 @@ +Run ReExportall to update JSON Exports + +http://guides.dataverse.org/en/4.19/admin/metadataexport.html?highlight=export#batch-exports-through-the-api \ No newline at end of file From 022306d178841a375983c7f0b7518f6f83be9d36 Mon Sep 17 00:00:00 2001 From: Peter Kiraly Date: Wed, 11 Mar 2020 17:43:44 +0100 Subject: [PATCH 131/157] #6736 Add Provenance Example File to Documentation. --- doc/sphinx-guides/source/_static/api/file-provenance.json | 1 + doc/sphinx-guides/source/api/native-api.rst | 2 ++ 2 files changed, 3 insertions(+) create mode 100644 doc/sphinx-guides/source/_static/api/file-provenance.json diff --git a/doc/sphinx-guides/source/_static/api/file-provenance.json b/doc/sphinx-guides/source/_static/api/file-provenance.json new file mode 100644 index 00000000000..6c823cdb5f3 --- /dev/null +++ b/doc/sphinx-guides/source/_static/api/file-provenance.json @@ -0,0 +1 @@ +{"prefix": {"pre_0": "http://www.w3.org/2001/XMLSchema", "s-prov": "http://s-prov/ns/#", "provone": "http://purl.dataone.org/provone/2015/01/15/ontology#", "vargen": "http://openprovenance.org/vargen#", "foaf": "http://xmlns.com/foaf/0.1/", "dcterms": "http://purl.org/dc/terms/", "tmpl": "http://openprovenance.org/tmpl#", "var": "http://openprovenance.org/var#", "vcard": "http://www.w3.org/2006/vcard/ns#", "swirrl": "http://project-dare.eu/ns#"}, "bundle": {"vargen:SessionSnapshot": {"prefix": {"s-prov": "http://s-prov/ns/#", "provone": "http://purl.dataone.org/provone/2015/01/15/ontology#", "vargen": "http://openprovenance.org/vargen#", "tmpl": "http://openprovenance.org/tmpl#", "var": "http://openprovenance.org/var#", "vcard": "http://www.w3.org/2006/vcard/ns#", "swirrl": "http://project-dare.eu/ns#"}, "entity": {"vargen:inData": {"swirrl:volumeId": {"$": "var:rawVolumeId", "type": "prov:QUALIFIED_NAME"}, "prov:type": {"$": "provone:Data", "type": "prov:QUALIFIED_NAME"}}, "vargen:inFile": {"prov:atLocation": {"$": "var:atLocation", "type": "prov:QUALIFIED_NAME"}, "s-prov:format": {"$": "var:format", "type": "prov:QUALIFIED_NAME"}, "s-prov:checksum": {"$": "var:checksum", "type": "prov:QUALIFIED_NAME"}}, "vargen:WorkData": {"swirrl:volumeId": {"$": "var:workVolumeId", "type": "prov:QUALIFIED_NAME"}, "prov:type": {"$": "provone:Data", "type": "prov:QUALIFIED_NAME"}}, "var:JupSnapshot": {"prov:generatedAt": {"$": "var:generatedAt", "type": "prov:QUALIFIED_NAME"}, "prov:atLocation": {"$": "var:repoUrl", "type": "prov:QUALIFIED_NAME"}, "s-prov:description": {"$": "var:description", "type": "prov:QUALIFIED_NAME"}, "prov:type": {"$": "swirrl:NotebookSnapshot", "type": "prov:QUALIFIED_NAME"}, "swirrl:sessionId": {"$": "var:sessionId", "type": "prov:QUALIFIED_NAME"}}}, "used": {"_:id1": {"prov:activity": "vargen:snapshot", "prov:entity": "var:Jupyter"}, "_:id2": {"prov:activity": "vargen:snapshot", "prov:entity": "vargen:WorkData"}, "_:id3": {"prov:activity": "vargen:snapshot", "prov:entity": "vargen:inData"}}, "wasDerivedFrom": {"_:id4": {"prov:usedEntity": "var:Jupyter", "prov:generatedEntity": "var:JupSnapshot"}}, "wasAssociatedWith": {"_:id5": {"prov:activity": "vargen:snapshot", "prov:agent": "var:snapAgent"}}, "actedOnBehalfOf": {"_:id6": {"prov:delegate": "var:snapAgent", "prov:responsible": "var:user"}}, "activity": {"vargen:snapshot": {"prov:atLocation": {"$": "var:method_path", "type": "prov:QUALIFIED_NAME"}, "tmpl:startTime": {"$": "var:startTime", "type": "prov:QUALIFIED_NAME"}, "tmpl:endTime": {"$": "var:endTime", "type": "prov:QUALIFIED_NAME"}}}, "wasGeneratedBy": {"_:id7": {"prov:activity": "vargen:snapshot", "prov:entity": "var:JupSnapshot"}}, "agent": {"var:user": {"vcard:uid": {"$": "var:name", "type": "prov:QUALIFIED_NAME"}, "swirrl:authMode": {"$": "var:authmode", "type": "prov:QUALIFIED_NAME"}, "swirrl:group": {"$": "var:group", "type": "prov:QUALIFIED_NAME"}, "prov:type": {"$": "prov:Person", "type": "prov:QUALIFIED_NAME"}}, "var:snapAgent": {"vcard:uid": {"$": "var:name_api", "type": "prov:QUALIFIED_NAME"}, "prov:type": {"$": "prov:SoftwareAgent", "type": "prov:QUALIFIED_NAME"}}}, "hadMember": {"_:id8": {"prov:collection": "vargen:inData", "prov:entity": "vargen:inFile"}}}}} \ No newline at end of file diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index fac4957bfb3..79ecdbad0b8 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -2190,6 +2190,8 @@ The fully expanded example above (without environment variables) looks like this curl -H "X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx" -X POST "https://demo.dataverse.org/api/files/:persistentId/prov-freeform?persistentId=doi:10.5072/FK2/AAA000" -H "Content-type:application/json" --upload-file provenance.json +See a sample JSON file :download:`file-provenance.json <../_static/api/file-provenance.json>` from http://openprovenance.org (c.f. Huynh, Trung Dong and Moreau, Luc (2014) ProvStore: a public provenance repository. At 5th International Provenance and Annotation Workshop (IPAW'14), Cologne, Germany, 09-13 Jun 2014. pp. 275-277). + Delete Provenance JSON for an uploaded file ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ From 00c62d20d7d5aa2d80d7897693b544c7bf2d7f1b Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 12 Mar 2020 09:48:01 -0400 Subject: [PATCH 132/157] retry to give S3 time to 'index' new objects eventual consistency... --- .../iq/dataverse/dataaccess/S3AccessIO.java | 27 ++++++++++++++----- 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 96ce92754a4..b65739915eb 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -213,14 +213,29 @@ public void open(DataAccessOption... options) throws IOException { } else if (dvObject instanceof Dataverse) { throw new IOException("Data Access: Storage driver does not support dvObject type Dataverse yet"); } else { - //Direct access, e.g. for external upload - no associated DVobject yet, but we want to be abel to get the size + // Direct access, e.g. for external upload - no associated DVobject yet, but we want to be able to get the size + // With small files, it looks like we may call before S3 says it exists, so try some retries before failing if(key!=null) { ObjectMetadata objectMetadata = null; - try { - objectMetadata = s3.getObjectMetadata(bucketName, key); - } catch (SdkClientException sce) { - throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); - } + int retries = 3; + while(retries > 0) { + try { + objectMetadata = s3.getObjectMetadata(bucketName, key); + retries = 0; + } catch (SdkClientException sce) { + if(retries > 1) { + retries--; + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + e.printStackTrace(); + } + logger.warning("Retrying after: " + sce.getMessage()); + } else { + throw new IOException("Cannot get S3 object " + key + " ("+sce.getMessage()+")"); + } + } + } this.setSize(objectMetadata.getContentLength()); }else { throw new IOException("Data Access: Invalid DvObject type"); From 2d28a71d7ca1a2afeec3359e553a01001d38e970 Mon Sep 17 00:00:00 2001 From: landreev Date: Fri, 13 Mar 2020 11:44:37 -0400 Subject: [PATCH 133/157] cors configuration example Added an example of an S3 bucket CORS configuration. Mostly to emphasize that it is required. --- .../source/developers/big-data-support.rst | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/developers/big-data-support.rst b/doc/sphinx-guides/source/developers/big-data-support.rst index a64dbb07ea1..c1c2969a60a 100644 --- a/doc/sphinx-guides/source/developers/big-data-support.rst +++ b/doc/sphinx-guides/source/developers/big-data-support.rst @@ -30,7 +30,26 @@ At present, one potential drawback for direct-upload is that files are only part ``./asadmin create-jvm-options "-Ddataverse.files..ingestsizelimit="`` -One additional step that is required to enable direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. +**IMPORTANT:** One additional step that is required to enable direct download to work with previewers is to allow cross site (CORS) requests on your S3 store. +The example below shows how to enable the minimum needed CORS rules on a bucket using the AWS CLI command line tool. Note that you may need to add more methods and/or locations, if you also need to support certain previewers and external tools. + +``aws s3api put-bucket-cors --bucket --cors-configuration file://cors.json`` + +with the contents of the file cors.json as follows: + +.. code-block:: json + + { + "CORSRules": [ + { + "AllowedOrigins": ["https://"], + "AllowedHeaders": ["*"], + "AllowedMethods": ["PUT", "GET"] + } + ] + } + +Alternatively, you can enable CORS using the AWS S3 web interface, using json-encoded rules as in the example above. Since the direct upload mechanism creates the final file rather than an intermediate temporary file, user actions, such as neither saving or canceling an upload session before closing the browser page, can leave an abandoned file in the store. The direct upload mechanism attempts to use S3 Tags to aid in identifying/removing such files. Upon upload, files are given a "dv-status":"temp" tag which is removed when the dataset changes are saved and the new file(s) are added in Dataverse. Note that not all S3 implementations support Tags: Minio does not. WIth such stores, direct upload works, but Tags are not used. From ab4f6c9fb40fbeb0d7c62527313437dd58714d05 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 13 Mar 2020 14:38:01 -0400 Subject: [PATCH 134/157] fix duplicate id issue with direct upload --- .../java/edu/harvard/iq/dataverse/DatasetPage.java | 10 ---------- src/main/webapp/dataset.xhtml | 1 - 2 files changed, 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index b13200b0dde..0ece7e9c4c2 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -2982,16 +2982,6 @@ public void setLinkingDataverseErrorMessage(String linkingDataverseErrorMessage) this.linkingDataverseErrorMessage = linkingDataverseErrorMessage; } - UIInput selectedLinkingDataverseMenu; - - public UIInput getSelectedDataverseMenu() { - return selectedLinkingDataverseMenu; - } - - public void setSelectedDataverseMenu(UIInput selectedDataverseMenu) { - this.selectedLinkingDataverseMenu = selectedDataverseMenu; - } - private Boolean saveLink(Dataverse dataverse){ boolean retVal = true; if (readOnly) { diff --git a/src/main/webapp/dataset.xhtml b/src/main/webapp/dataset.xhtml index db7620bab71..d907aa5aceb 100644 --- a/src/main/webapp/dataset.xhtml +++ b/src/main/webapp/dataset.xhtml @@ -1363,7 +1363,6 @@
Date: Mon, 16 Mar 2020 09:22:49 -0400 Subject: [PATCH 135/157] A *very* simple solution for the undetected database save fail in ingest. (is it too simple? am I missing something??) #6660 --- .../harvard/iq/dataverse/DataFileServiceBean.java | 13 +++++++++++++ .../iq/dataverse/ingest/IngestServiceBean.java | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java index 54a88c27d91..448218c93cd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/DataFileServiceBean.java @@ -29,6 +29,8 @@ import java.util.logging.Logger; import javax.ejb.EJB; import javax.ejb.Stateless; +import javax.ejb.TransactionAttribute; +import javax.ejb.TransactionAttributeType; import javax.inject.Named; import javax.persistence.EntityManager; import javax.persistence.NoResultException; @@ -942,6 +944,17 @@ public DataFile save(DataFile dataFile) { } } + @TransactionAttribute(TransactionAttributeType.REQUIRES_NEW) + public DataFile saveInTransaction(DataFile dataFile) { + + if (dataFile.isMergeable()) { + DataFile savedDataFile = em.merge(dataFile); + return savedDataFile; + } else { + throw new IllegalArgumentException("This DataFile object has been set to NOT MERGEABLE; please ensure a MERGEABLE object is passed to the save method."); + } + } + private void msg(String m){ System.out.println(m); } diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index b06877df2ae..7922bcdd452 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -955,7 +955,7 @@ public boolean ingestAsTabular(Long datafile_id) { throw new EJBException("Deliberate database save failure"); } */ - dataFile = fileService.save(dataFile); + dataFile = fileService.saveInTransaction(dataFile); databaseSaveSuccessful = true; logger.fine("Ingest (" + dataFile.getFileMetadata().getLabel() + "."); From 34df93c195dc7274a5bcb9540424d4282b3ac9a3 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 16 Mar 2020 14:06:15 -0400 Subject: [PATCH 136/157] One extra dataFileService.save(), to ensure there's no optimistic lock after a successful update. (#6660) --- .../edu/harvard/iq/dataverse/ingest/IngestServiceBean.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 7922bcdd452..c6ca819f4fd 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -958,7 +958,7 @@ public boolean ingestAsTabular(Long datafile_id) { dataFile = fileService.saveInTransaction(dataFile); databaseSaveSuccessful = true; - logger.fine("Ingest (" + dataFile.getFileMetadata().getLabel() + "."); + logger.info("Ingest (" + dataFile.getFileMetadata().getLabel() + "."); if (additionalData != null) { // remove the extra tempfile, if there was one: @@ -996,7 +996,7 @@ public boolean ingestAsTabular(Long datafile_id) { // and we want to save the original of the ingested file: try { dataAccess.backupAsAux(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); - logger.fine("Saved the ingested original as a backup aux file "+FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); + logger.info("Saved the ingested original as a backup aux file "+FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); } catch (IOException iox) { logger.warning("Failed to save the ingested original! " + iox.getMessage()); } @@ -1005,6 +1005,9 @@ public boolean ingestAsTabular(Long datafile_id) { dataAccess.savePath(Paths.get(tabFile.getAbsolutePath())); // Reset the file size: dataFile.setFilesize(dataAccess.getSize()); + + dataFile = fileService.save(dataFile); + logger.info("saved data file after updating the size"); // delete the temp tab-file: tabFile.delete(); From 4e70dd6ffa8a9ef6618a0ce4e60e44336ff90a09 Mon Sep 17 00:00:00 2001 From: Don Sizemore Date: Mon, 16 Mar 2020 14:22:15 -0400 Subject: [PATCH 137/157] #6742 update EC2 documentation --- .../source/developers/deployment.rst | 15 ++- .../source/developers/testing.rst | 29 +++-- scripts/installer/ec2-create-instance.sh | 107 ++++++++++++++---- 3 files changed, 111 insertions(+), 40 deletions(-) diff --git a/doc/sphinx-guides/source/developers/deployment.rst b/doc/sphinx-guides/source/developers/deployment.rst index 9532e7c769f..5e830bfde5b 100755 --- a/doc/sphinx-guides/source/developers/deployment.rst +++ b/doc/sphinx-guides/source/developers/deployment.rst @@ -82,23 +82,26 @@ Download and Run the "Create Instance" Script Once you have done the configuration above, you are ready to try running the "ec2-create-instance.sh" script to spin up Dataverse in AWS. -Download :download:`ec2-create-instance.sh <../../../../scripts/installer/ec2-create-instance.sh>` and put it somewhere reasonable. For the purpose of these instructions we'll assume it's in the "Downloads" directory in your home directory. +Download :download:`ec2-create-instance.sh` and put it somewhere reasonable. For the purpose of these instructions we'll assume it's in the "Downloads" directory in your home directory. -ec2-create-instance accepts a number few command-line switches: +To run it with default values you just need the script, but you may also want a current copy of the ansible :download:`group vars`_ file. + +ec2-create-instance accepts a number of command-line switches, including: * -r: GitHub Repository URL (defaults to https://github.com/IQSS/dataverse.git) * -b: branch to build (defaults to develop) * -p: pemfile directory (defaults to $HOME) * -g: Ansible GroupVars file (if you wish to override role defaults) +* -h: help (displays usage for each available option) ``bash ~/Downloads/ec2-create-instance.sh -b develop -r https://github.com/scholarsportal/dataverse.git -g main.yml`` -Now you will need to wait around 15 minutes until the deployment is finished. Eventually, the output should tell you how to access the installation of Dataverse in a web browser or via ssh. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You can also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 . +You will need to wait for 15 minutes or so until the deployment is finished, longer if you've enabled sample data and/or the API test suite. Eventually, the output should tell you how to access the installation of Dataverse in a web browser or via SSH. It will also provide instructions on how to delete the instance when you are finished with it. Please be aware that AWS charges per minute for a running instance. You may also delete your instance from https://console.aws.amazon.com/console/home?region=us-east-1 . -Caveats -~~~~~~~ +Caveat Recipiens +~~~~~~~~~~~~~~~~ -Please note that while the script should work fine on newish branches, older branches that have different dependencies such as an older version of Solr may not produce a working Dataverse installation. Your mileage may vary. +Please note that while the script should work well on new-ish branches, older branches that have different dependencies such as an older version of Solr may not produce a working Dataverse installation. Your mileage may vary. ---- diff --git a/doc/sphinx-guides/source/developers/testing.rst b/doc/sphinx-guides/source/developers/testing.rst index 82a4e22ebed..2894c457d85 100755 --- a/doc/sphinx-guides/source/developers/testing.rst +++ b/doc/sphinx-guides/source/developers/testing.rst @@ -108,22 +108,33 @@ Unfortunately, the term "integration tests" can mean different things to differe Running the Full API Test Suite Using EC2 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -To run the API test suite on EC2 you should first follow the steps in the :doc:`deployment` section to get set up for AWS in general and EC2 in particular. +To run the API test suite in an EC2 instance you should first follow the steps in the :doc:`deployment` section to get set up for AWS in general and EC2 in particular. -Then read the instructions in https://github.com/IQSS/dataverse-sample-data for EC2 but be sure to make the adjustments below. +You may always retrieve a current copy of the ec2-create-instance.sh script and accompanying group_var.yml file from the `dataverse-ansible repo`_: -Edit ``ec2config.yaml`` to change ``test_suite`` to ``true``. +- `ec2-create-instance.sh`_ +- `main.yml`_ -Pass in the repo and branch you are testing. You should also specify a local directory where server.log and other useful information will be written so you can start debugging any failures. +Edit ``main.yml`` to set the desired GitHub repo, branch, and to ensure that the API test suite is enabled: + +- ``dataverse_repo: https://github.com/IQSS/dataverse.git`` +- ``dataverse_branch: develop`` +- ``dataverse.api.test_suite: true`` +- ``dataverse.sampledata.enabled: true`` + +If you wish, you may pass the local path of a logging directory, which will tell ec2-create-instance.sh to `grab glassfish, maven and other logs`_ for your review. + +Finally, run the script: .. code-block:: bash - export REPO=https://github.com/IQSS/dataverse.git - export BRANCH=123-my-branch - export LOGS=/tmp/123 + $ ./ec2-create-instance.sh -g main.yml -l log_dir + +Near the beginning and at the end of the ec2-create-instance.sh output you will see instructions for connecting to the instance via SSH. If you are actively working on a branch and want to refresh the warfile after each commit, you may wish to call a `redeploy.sh`_ script placed by the Ansible role, which will do a "git pull" against your branch, build the warfile, deploy the warfile, then restart glassfish. By default this script is written to /tmp/dataverse/redeploy.sh. You may invoke the script by appending it to the SSH command in ec2-create's output: + +.. code-block:: bash - mkdir $LOGS - ./ec2-create-instance.sh -g ec2config.yaml -r $REPO -b $BRANCH -l $LOGS + $ ssh -i your_pem.pem user@ec2-host.aws.com /tmp/dataverse/redeploy.sh Running the full API test suite using Docker ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/scripts/installer/ec2-create-instance.sh b/scripts/installer/ec2-create-instance.sh index 63e9ac21bc6..81fdd5940c1 100755 --- a/scripts/installer/ec2-create-instance.sh +++ b/scripts/installer/ec2-create-instance.sh @@ -3,21 +3,28 @@ # For docs, see the "Deployment" page in the Dev Guide. # repo and branch defaults -REPO_URL='https://github.com/IQSS/dataverse.git' -BRANCH='develop' +REPO_URL_DEFAULT='https://github.com/IQSS/dataverse.git' +BRANCH_DEFAULT='develop' PEM_DEFAULT=${HOME} +AWS_AMI_DEFAULT='ami-9887c6e7' usage() { - echo "Usage: $0 -b -r -p -g " 1>&2 + echo "Usage: $0 -b -r -p -g -a -i aws_image -s aws_size -t aws_tag -l local_log_path" 1>&2 echo "default branch is develop" echo "default repo is https://github.com/IQSS/dataverse" echo "default .pem location is ${HOME}" echo "example group_vars may be retrieved from https://raw.githubusercontent.com/IQSS/dataverse-ansible/master/defaults/main.yml" + echo "default AWS AMI ID is $AWS_AMI_DEFAULT" + echo "default AWS size is t2.medium" + echo "local log path" exit 1 } -while getopts ":r:b:g:p:" o; do +while getopts ":a:r:b:g:p:i:s:t:l:" o; do case "${o}" in + a) + DA_BRANCH=${OPTARG} + ;; r) REPO_URL=${OPTARG} ;; @@ -30,32 +37,74 @@ while getopts ":r:b:g:p:" o; do p) PEM_DIR=${OPTARG} ;; + i) + AWS_IMAGE=${OPTARG} + ;; + s) + AWS_SIZE=${OPTARG} + ;; + t) + TAG=${OPTARG} + ;; + l) + LOCAL_LOG_PATH=${OPTARG} + ;; *) usage ;; esac done -# test for user-supplied conf files +# test for ansible group_vars if [ ! -z "$GRPVRS" ]; then GVFILE=$(basename "$GRPVRS") GVARG="-e @$GVFILE" echo "using $GRPVRS for extra vars" fi +# test for CLI args if [ ! -z "$REPO_URL" ]; then GVARG+=" -e dataverse_repo=$REPO_URL" - echo "using $REPO_URL" + echo "using repo $REPO_URL" fi if [ ! -z "$BRANCH" ]; then GVARG+=" -e dataverse_branch=$BRANCH" - echo "building $BRANCH" + echo "building branch $BRANCH" fi +# The AMI ID may change in the future and the way to look it up is with the following command, which takes a long time to run: +# aws ec2 describe-images --owners 'aws-marketplace' --filters 'Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce' --query 'sort_by(Images, &CreationDate)[-1].[ImageId]' --output 'text' +# To use an AMI, one must subscribe to it via the AWS GUI. +# AMI IDs are specific to the region. + +if [ ! -z "$AWS_IMAGE" ]; then + AMI_ID=$AWS_IMAGE +else + AMI_ID="$AWS_AMI_DEFAULT" +fi +echo "using $AMI_ID" + +if [ ! -z "$AWS_SIZE" ]; then + SIZE=$AWS_SIZE +else + SIZE="t2.medium" +fi +echo "using $SIZE" + +if [ ! -z "$TAG" ]; then + TAGARG="--tag-specifications ResourceType=instance,Tags=[{Key=name,Value=$TAG}]" + echo "using tag $TAG" +fi + +# default to dataverse-ansible/master +if [ -z "$DA_BRANCH" ]; then + DA_BRANCH="master" +fi + +# ansible doesn't care about pem_dir (yet) if [ -z "$PEM_DIR" ]; then PEM_DIR="$PEM_DEFAULT" - echo "using $PEM_DIR" fi AWS_CLI_VERSION=$(aws --version) @@ -95,22 +144,12 @@ else exit 1 fi -# The AMI ID may change in the future and the way to look it up is with the -# following command, which takes a long time to run: -# -# aws ec2 describe-images --owners 'aws-marketplace' --filters 'Name=product-code,Values=aw0evgkw8e5c1q413zgy5pjce' --query 'sort_by(Images, &CreationDate)[-1].[ImageId]' --output 'text' -# -# To use this AMI, we subscribed to it from the AWS GUI. -# AMI IDs are specific to the region. -AMI_ID='ami-9887c6e7' -# Smaller than medium lead to Maven and Solr problems. -SIZE='t2.medium' echo "Creating EC2 instance" # TODO: Add some error checking for "ec2 run-instances". -INSTANCE_ID=$(aws ec2 run-instances --image-id $AMI_ID --security-groups $SECURITY_GROUP --count 1 --instance-type $SIZE --key-name $PEM_DIR/$KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true } } ]' | tr -d \") +INSTANCE_ID=$(aws ec2 run-instances --image-id $AMI_ID --security-groups $SECURITY_GROUP $TAGARG --count 1 --instance-type $SIZE --key-name $PEM_DIR/$KEY_NAME --query 'Instances[0].InstanceId' --block-device-mappings '[ { "DeviceName": "/dev/sda1", "Ebs": { "DeleteOnTermination": true } } ]' | tr -d \") echo "Instance ID: "$INSTANCE_ID -echo "giving instance 30 seconds to wake up..." -sleep 30 +echo "giving instance 60 seconds to wake up..." +sleep 60 echo "End creating EC2 instance" PUBLIC_DNS=$(aws ec2 describe-instances --instance-ids $INSTANCE_ID --query "Reservations[*].Instances[*].[PublicDnsName]" --output text) @@ -132,18 +171,36 @@ ssh -T -i $PEM_FILE -o 'StrictHostKeyChecking no' -o 'UserKnownHostsFile=/dev/nu sudo yum -y install epel-release sudo yum -y install https://releases.ansible.com/ansible/rpm/release/epel-7-x86_64/ansible-2.7.9-1.el7.ans.noarch.rpm sudo yum -y install git nano -git clone https://github.com/IQSS/dataverse-ansible.git dataverse +git clone -b $DA_BRANCH https://github.com/IQSS/dataverse-ansible.git dataverse export ANSIBLE_ROLES_PATH=. -echo $extra_vars ansible-playbook -v -i dataverse/inventory dataverse/dataverse.pb --connection=local $GVARG EOF +if [ ! -z "$LOCAL_LOG_PATH" ]; then + echo "copying logs to $LOCAL_LOG_PATH." + # 1 accept SSH keys + ssh-keyscan ${PUBLIC_DNS} >> ~/.ssh/known_hosts + # 2 logdir should exist + mkdir -p $LOCAL_LOG_PATH + # 3 grab logs for local processing in jenkins + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/target/site $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/target/surefire-reports $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" centos@$PUBLIC_DNS:/usr/local/glassfish4/glassfish/domains/domain1/logs/server* $LOCAL_LOG_PATH/ + # 4 grab mvn.out + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/mvn.out $LOCAL_LOG_PATH/ + # 5 jacoco + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/target/coverage-it $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/target/*.exec $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/target/classes $LOCAL_LOG_PATH/ + rsync -av -e "ssh -i $PEM_FILE" --ignore-missing-args centos@$PUBLIC_DNS:/tmp/dataverse/src $LOCAL_LOG_PATH/ +fi + # Port 8080 has been added because Ansible puts a redirect in place # from HTTP to HTTPS and the cert is invalid (self-signed), forcing # the user to click through browser warnings. -CLICKABLE_LINK="http://${PUBLIC_DNS}:8080" +CLICKABLE_LINK="http://${PUBLIC_DNS}" echo "To ssh into the new instance:" echo "ssh -i $PEM_FILE $USER_AT_HOST" -echo "Branch \"$BRANCH\" from $REPO_URL has been deployed to $CLICKABLE_LINK" +echo "Branch $BRANCH from $REPO_URL has been deployed to $CLICKABLE_LINK" echo "When you are done, please terminate your instance with:" echo "aws ec2 terminate-instances --instance-ids $INSTANCE_ID" From d4bfae0d5373f690290003012100f276b69bc084 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Mon, 16 Mar 2020 18:39:17 -0400 Subject: [PATCH 138/157] cleaned up logging messages (#6660) --- .../harvard/iq/dataverse/ingest/IngestServiceBean.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index c6ca819f4fd..5e84ecb0155 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -958,7 +958,7 @@ public boolean ingestAsTabular(Long datafile_id) { dataFile = fileService.saveInTransaction(dataFile); databaseSaveSuccessful = true; - logger.info("Ingest (" + dataFile.getFileMetadata().getLabel() + "."); + logger.fine("Ingest (" + dataFile.getFileMetadata().getLabel() + "."); if (additionalData != null) { // remove the extra tempfile, if there was one: @@ -982,7 +982,7 @@ public boolean ingestAsTabular(Long datafile_id) { } if (!databaseSaveSuccessful) { - logger.warning("Ingest failure (!databaseSaveSuccessful)."); + logger.warning("Ingest failure (failed to save the tabular data in the database; file left intact as uploaded)."); return false; } @@ -996,7 +996,7 @@ public boolean ingestAsTabular(Long datafile_id) { // and we want to save the original of the ingested file: try { dataAccess.backupAsAux(FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); - logger.info("Saved the ingested original as a backup aux file "+FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); + logger.fine("Saved the ingested original as a backup aux file "+FileUtil.SAVED_ORIGINAL_FILENAME_EXTENSION); } catch (IOException iox) { logger.warning("Failed to save the ingested original! " + iox.getMessage()); } @@ -1007,7 +1007,7 @@ public boolean ingestAsTabular(Long datafile_id) { dataFile.setFilesize(dataAccess.getSize()); dataFile = fileService.save(dataFile); - logger.info("saved data file after updating the size"); + logger.fine("saved data file after updating the size"); // delete the temp tab-file: tabFile.delete(); From b1221c1cd03b19f258df70acde1370e49392a98d Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 18 Mar 2020 15:03:04 -0400 Subject: [PATCH 139/157] update script with version changes (per discussion) --- src/main/webapp/editFilesFragment.xhtml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 57d760f6a56..5d431ecbac7 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -11,7 +11,7 @@ xmlns:o="http://omnifaces.org/ui" xmlns:iqbs="http://xmlns.jcp.org/jsf/composite/iqbs"> - + From be03eadb0f705d07d0c37ab57026b7d12d171356 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 18 Mar 2020 15:58:49 -0400 Subject: [PATCH 140/157] limit parallel requests (to 4 at the moment) --- src/main/webapp/resources/js/fileupload.js | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index b177fed7df7..5a18b8b7662 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -1,6 +1,7 @@ var fileList = []; var observer2=null; var datasetId=null; +var filesInProgress=0; function setupDirectUpload(enabled, theDatasetId) { if(enabled) { @@ -40,13 +41,17 @@ function setupDirectUpload(enabled, theDatasetId) { function queueFileForDirectUpload(file, datasetId) { if(fileList.length === 0) {uploadWidgetDropRemoveMsg();} fileList.push(file); - - requestDirectUploadUrl(); + if(filesInProgress <5 ) { + filesInProgress= filesInProgress+1; + requestDirectUploadUrl(); + + } } function uploadFileDirectly(url, storageId) { //Pick a pending file var file = fileList.pop(); + filesInProgress=filesInProgress-1; $('.ui-fileupload-progress').html(''); $('.ui-fileupload-progress').append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); $.ajax({ @@ -165,6 +170,11 @@ function directUploadFinished() { observer.disconnect(); observer=null; } + } else { + if(filesInProgress <5 ) { + filesInProgress= filesInProgress+1; + requestDirectUploadUrl(); + } } } From c2cf2759f999d97533c92c3013a9a97601b126a8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Thu, 19 Mar 2020 16:57:16 -0400 Subject: [PATCH 141/157] handle progress bars, avoid calling AllUploadsFinished more than once --- src/main/webapp/resources/js/fileupload.js | 114 +++++++++++++-------- 1 file changed, 71 insertions(+), 43 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 5a18b8b7662..cdc9af6d660 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -1,7 +1,14 @@ var fileList = []; var observer2=null; var datasetId=null; +//How many files have started being processed but aren't yet being uploaded var filesInProgress=0; +//The # of the current file being processed (total number of files for which upload has at least started) +var curFile=0; +//The number of upload ids that have been assigned in the files table +var fileUpId=0; +//How many files are completely done +var filesFinished=0; function setupDirectUpload(enabled, theDatasetId) { if(enabled) { @@ -41,19 +48,37 @@ function setupDirectUpload(enabled, theDatasetId) { function queueFileForDirectUpload(file, datasetId) { if(fileList.length === 0) {uploadWidgetDropRemoveMsg();} fileList.push(file); - if(filesInProgress <5 ) { + //Fire off the first 4 to start (0,1,2,3) + if(filesInProgress < 4 ) { filesInProgress= filesInProgress+1; requestDirectUploadUrl(); - } } function uploadFileDirectly(url, storageId) { - //Pick a pending file - var file = fileList.pop(); + //Pick the 'first-in' pending file + var file = fileList.shift(); + //This appears to be the earliest point when the file table has been populated, and, since we don't know how many table entries have had ids added already, we check + var filerows = $('.ui-fileupload-files .ui-fileupload-row'); + //Add an id attribute to each entry so we can later match progress and errors with the right entry + for(i=0;i< filerows.length;i++) { + var upid=filerows[i].getAttribute('upid'); + if(typeof upid === "undefined" || upid === null || upid === '') { + filerows[i].setAttribute('upid', fileUpId); + fileUpId = fileUpId+1; + } + } + //Get the list of files to upload + var files = $('.ui-fileupload-files'); + //Find the corresponding row (assumes that the file order and the order of rows is the same) + var fileNode = files.find("[upid='"+curFile+"']"); + //Increment count of files being processed + curFile=curFile+1; + //Decrement number queued for processing filesInProgress=filesInProgress-1; - $('.ui-fileupload-progress').html(''); - $('.ui-fileupload-progress').append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); + var progBar = fileNode.find('.ui-fileupload-progress'); + progBar.html(''); + progBar.append($('').attr('class', 'ui-progressbar ui-widget ui-widget-content ui-corner-all')); $.ajax({ url: url, headers: {"x-amz-tagging":"dv-state=temp"}, @@ -62,7 +87,7 @@ function uploadFileDirectly(url, storageId) { cache: false, processData: false, success: function () { - reportUpload(storageId, file) + reportUpload(storageId, file) }, error: function(jqXHR, textStatus, errorThrown) { console.log('Failure: ' + jqXHR.status); @@ -74,7 +99,7 @@ function uploadFileDirectly(url, storageId) { myXhr.upload.addEventListener('progress', function(e) { if(e.lengthComputable) { var doublelength = 2 * e.total; - $('progress').attr({ + progBar.children('progress').attr({ value:e.loaded, max:doublelength }); @@ -87,11 +112,11 @@ function uploadFileDirectly(url, storageId) { } function reportUpload(storageId, file){ - + getMD5( file, prog => {console.log("Progress: " + prog); - + var current = 1 + prog; $('progress').attr({ value:current, @@ -119,36 +144,36 @@ function removeErrors() { var observer=null; function uploadStarted() { - // If this is not the first upload, remove error messages since - // the upload of any files that failed will be tried again. - removeErrors(); - var curId=0; - //Find the upload table body - var files = $('.ui-fileupload-files .ui-fileupload-row'); - //Add an id attribute to each entry so we can later match errors with the right entry - for(i=0;i< files.length;i++) { - files[i].setAttribute('upid', curId); - curId = curId+1; - } - //Setup an observer to watch for additional rows being added - var config={childList: true}; - var callback = function(mutations) { - //Add an id attribute to all new entries - mutations.forEach(function(mutation) { - for(i=0; i Date: Thu, 19 Mar 2020 17:38:58 -0400 Subject: [PATCH 142/157] Add support for drag and drop --- src/main/webapp/resources/js/fileupload.js | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index cdc9af6d660..1f27f0717f0 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -15,17 +15,26 @@ function setupDirectUpload(enabled, theDatasetId) { datasetId=theDatasetId; $('.ui-fileupload-upload').hide(); $('.ui-fileupload-cancel').hide(); + //Catch files entered via upload dialog box. Since this 'select' widget is replaced by PF, we need to add a listener again when it is replaced var fileInput=document.getElementById('datasetForm:fileUpload_input'); fileInput.addEventListener('change', function(event) { - fileList=[]; - for(var i=0;i Date: Fri, 20 Mar 2020 10:16:42 -0400 Subject: [PATCH 143/157] cut/paste error and adding an error handler --- src/main/webapp/resources/js/fileupload.js | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 1f27f0717f0..c6d2ad0d302 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -24,10 +24,12 @@ function setupDirectUpload(enabled, theDatasetId) { } }, {once:false}); //Add support for drag and drop. Since the fileUploadForm is not replaced by PF, catching changes with a mutationobserver isn't needed - var fileDropWidget=document.getElementById('datasetForm:fileUpload'); fileDropWidget.addEventListener('drop', function(event) { console.log('Drop!'); - fileDropWidget.addEventListener('drop', function(event) { console.log('Drop!'); - fileList=[]; - for(var i=0;i Date: Fri, 20 Mar 2020 10:19:50 -0400 Subject: [PATCH 144/157] more error handling --- src/main/webapp/editFilesFragment.xhtml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/webapp/editFilesFragment.xhtml b/src/main/webapp/editFilesFragment.xhtml index 5d431ecbac7..7f7d3e5c594 100644 --- a/src/main/webapp/editFilesFragment.xhtml +++ b/src/main/webapp/editFilesFragment.xhtml @@ -521,8 +521,8 @@
- - + + From 9668c5f3fafc5acd07d2eb2b70cff6bf8fb2b9b8 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 20 Mar 2020 12:40:43 -0400 Subject: [PATCH 145/157] increasing retries and adding logging for testing --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index b65739915eb..9ebed03dc67 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -217,16 +217,17 @@ public void open(DataAccessOption... options) throws IOException { // With small files, it looks like we may call before S3 says it exists, so try some retries before failing if(key!=null) { ObjectMetadata objectMetadata = null; - int retries = 3; + int retries = 20; while(retries > 0) { try { objectMetadata = s3.getObjectMetadata(bucketName, key); + logger.warning("Success for key: " + key + " after " + ((20-retries)*3) + " seconds"); retries = 0; } catch (SdkClientException sce) { if(retries > 1) { retries--; try { - Thread.sleep(1000); + Thread.sleep(3000); } catch (InterruptedException e) { e.printStackTrace(); } From 21f736122438e98562bc6e998d41785d424533e1 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 20 Mar 2020 15:24:06 -0400 Subject: [PATCH 146/157] add error handling and pre/post S3 logging --- src/main/webapp/resources/js/fileupload.js | 46 +++++++++++++++------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index c6d2ad0d302..7e2622c54fa 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -69,6 +69,7 @@ function queueFileForDirectUpload(file, datasetId) { function uploadFileDirectly(url, storageId) { //Pick the 'first-in' pending file var file = fileList.shift(); + console.log('Uploading ' + file.name + ' as ' +storageId + ' to ' + url); //This appears to be the earliest point when the file table has been populated, and, since we don't know how many table entries have had ids added already, we check var filerows = $('.ui-fileupload-files .ui-fileupload-row'); //Add an id attribute to each entry so we can later match progress and errors with the right entry @@ -84,6 +85,7 @@ function uploadFileDirectly(url, storageId) { //Find the corresponding row (assumes that the file order and the order of rows is the same) var fileNode = files.find("[upid='"+curFile+"']"); //Increment count of files being processed + var thisFile=curFile; curFile=curFile+1; //Decrement number queued for processing filesInProgress=filesInProgress-1; @@ -101,7 +103,7 @@ function uploadFileDirectly(url, storageId) { reportUpload(storageId, file) }, error: function(jqXHR, textStatus, errorThrown) { - uploadFailure(); + uploadFailure(jqXHR, thisFile); }, xhr: function() { var myXhr = $.ajaxSettings.xhr(); @@ -122,10 +124,10 @@ function uploadFileDirectly(url, storageId) { } function reportUpload(storageId, file){ - + console.log('S3 Upload complete for ' + file.name + ' : ' + storageId); getMD5( file, - prog => {console.log("Progress: " + prog); + prog => { var current = 1 + prog; $('progress').attr({ @@ -216,7 +218,7 @@ function directUploadFinished() { } } -function uploadFailure(fileUpload) { +function uploadFailure(jqXHR, upid, filename) { // This handles HTTP errors (non-20x reponses) such as 0 (no connection at all), 413 (Request too large), // and 504 (Gateway timeout) where the upload call to the server fails (the server doesn't receive the request) // It notifies the user and provides info about the error (status, statusText) @@ -225,20 +227,34 @@ function uploadFailure(fileUpload) { // from the call stack instead (arguments to the fail() method that calls onerror() that calls this function //Retrieve the error number (status) and related explanation (statusText) - var status = arguments.callee.caller.caller.arguments[1].jqXHR.status; - var statusText = arguments.callee.caller.caller.arguments[1].jqXHR.statusText; + var status = null; + var statusText =null; + + // There are various metadata available about which file the error pertains to + // including the name and size. + // However, since the table rows created by PrimeFaces only show name and approximate size, + // these may not uniquely identify the affected file. Therefore, we set a unique upid attribute + // in uploadStarted (and the MutationObserver there) and look for that here. The files array has + // only one element and that element includes a description of the row involved, including it's upid. + + var name = null; + var id=null; + + if(!(typeof jqXHR !=='undefined')) { + status = jqXHR.status; + statusText=jqXHR.statusText; + id = upid; + name=filename; + } else { + status=arguments.callee.caller.caller.arguments[1].jqXHR.status; + statusText = arguments.callee.caller.caller.arguments[1].jqXHR.statusText; + name = arguments.callee.caller.caller.arguments[1].files[0].name; + id = arguments.callee.caller.caller.arguments[1].files[0].row[0].attributes.upid.value; + } + //statusText for error 0 is the unhelpful 'error' if(status == 0) statusText='Network Error'; - // There are various metadata available about which file the error pertains to - // including the name and size. - // However, since the table rows created by PrimeFaces only show name and approximate size, - // these may not uniquely identify the affected file. Therefore, we set a unique upid attribute - // in uploadStarted (and the MutationObserver there) and look for that here. The files array has - // only one element and that element includes a description of the row involved, including it's upid. - - var name = arguments.callee.caller.caller.arguments[1].files[0].name; - var id = arguments.callee.caller.caller.arguments[1].files[0].row[0].attributes.upid.value; //Log the error console.log('Upload error:' + name + ' upid=' + id + ', Error ' + status + ': ' + statusText ); //Find the table From 8eba2c1db53c310b2ae832b747e2eb5e70315546 Mon Sep 17 00:00:00 2001 From: Leonid Andreev Date: Fri, 20 Mar 2020 16:55:44 -0400 Subject: [PATCH 147/157] fixes the issue where the dataset directory is only created when a datafile is saved, but not when saving dataset-level files. (#6739) --- .../iq/dataverse/dataaccess/FileAccessIO.java | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java index d7a405d63c7..2b1a8d30918 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/FileAccessIO.java @@ -147,7 +147,6 @@ public void open (DataAccessOption... options) throws IOException { } else if (dvObject instanceof Dataset) { //This case is for uploading a dataset related auxiliary file //e.g. image thumbnails/metadata exports - //TODO: do we really need to do anything here? should we return the dataset directory? dataset = this.getDataset(); if (isReadAccess) { //TODO: Not necessary for dataset as there is no files associated with this @@ -229,10 +228,19 @@ public void saveInputStream(InputStream inputStream) throws IOException { @Override public Channel openAuxChannel(String auxItemTag, DataAccessOption... options) throws IOException { - + Path auxPath = getAuxObjectAsPath(auxItemTag); if (isWriteAccessRequested(options)) { + if (dvObject instanceof Dataset && !this.canWrite()) { + // If this is a dataset-level auxilary file (a cached metadata export, + // dataset logo, etc.) there's a chance that no "real" files + // have been saved for this dataset yet, and thus the filesystem + // directory does not exist yet. Let's force a proper .open() on + // this StorageIO, that will ensure it is created: + open(DataAccessOption.WRITE_ACCESS); + } + FileOutputStream auxOut = new FileOutputStream(auxPath.toFile()); if (auxOut == null) { @@ -287,7 +295,7 @@ public Path getAuxObjectAsPath(String auxItemTag) throws IOException { } String datasetDirectory = getDatasetDirectory(); - + if (dvObject.getStorageIdentifier() == null || "".equals(dvObject.getStorageIdentifier())) { throw new IOException("Data Access: No local storage identifier defined for this datafile."); } @@ -325,6 +333,10 @@ public void revertBackupAsAux(String auxItemTag) throws IOException { // this method copies a local filesystem Path into this DataAccess Auxiliary location: @Override public void savePathAsAux(Path fileSystemPath, String auxItemTag) throws IOException { + if (dvObject instanceof Dataset && !this.canWrite()) { + // see the comment in openAuxChannel() + open(DataAccessOption.WRITE_ACCESS); + } // quick Files.copy method: try { Path auxPath = getAuxObjectAsPath(auxItemTag); @@ -340,7 +352,10 @@ public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag, Lon @Override public void saveInputStreamAsAux(InputStream inputStream, String auxItemTag) throws IOException { - + if (dvObject instanceof Dataset && !this.canWrite()) { + // see the comment in openAuxChannel() + open(DataAccessOption.WRITE_ACCESS); + } // Since this is a local fileystem file, we can use the // quick NIO Files.copy method: From a15b83d84d022c42af64f6fec2f9308556992959 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 20 Mar 2020 17:44:06 -0400 Subject: [PATCH 148/157] tighten up on async issues create local vars from global ones so that value isn't changed by another async callback before we use it. --- src/main/webapp/resources/js/fileupload.js | 103 ++++++++++++--------- 1 file changed, 57 insertions(+), 46 deletions(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index 7e2622c54fa..c47ce524dc7 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -6,9 +6,16 @@ var filesInProgress=0; //The # of the current file being processed (total number of files for which upload has at least started) var curFile=0; //The number of upload ids that have been assigned in the files table -var fileUpId=0; +var getUpId = (function () { + var counter = -1; + return function () {counter += 1; return counter} +})(); //How many files are completely done -var filesFinished=0; +var finishFile = (function () { + var counter = 0; + return function () {counter += 1; return counter} +})(); + function setupDirectUpload(enabled, theDatasetId) { if(enabled) { @@ -23,10 +30,10 @@ function setupDirectUpload(enabled, theDatasetId) { queueFileForDirectUpload(fileInput.files[i], datasetId); } }, {once:false}); - //Add support for drag and drop. Since the fileUploadForm is not replaced by PF, catching changes with a mutationobserver isn't needed + //Add support for drag and drop. Since the fileUploadForm is not replaced by PF, catching changes with a mutationobserver isn't needed var fileDropWidget=document.getElementById('datasetForm:fileUpload'); fileDropWidget.addEventListener('drop', function(event) { - fileList=[]; + fileList=[]; for(var i=0;i= fileSize) { endCallback(null); return; @@ -336,5 +349,3 @@ function getMD5(blob, cbProgress) { }); }); } - - From 3a9e8117282c1e01338d4e39fa19500ae0551259 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Fri, 20 Mar 2020 18:44:53 -0400 Subject: [PATCH 149/157] remove debug log message --- .../edu/harvard/iq/dataverse/ingest/IngestServiceBean.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index c2a2a13c272..e7ccc7bf1b4 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -310,8 +310,7 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List Date: Mon, 23 Mar 2020 15:56:11 -0400 Subject: [PATCH 150/157] only warn if a retry occurred --- .../java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 9ebed03dc67..973fa8ee42a 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -221,7 +221,9 @@ public void open(DataAccessOption... options) throws IOException { while(retries > 0) { try { objectMetadata = s3.getObjectMetadata(bucketName, key); - logger.warning("Success for key: " + key + " after " + ((20-retries)*3) + " seconds"); + if(retries != 20) { + logger.warning("Success for key: " + key + " after " + ((20-retries)*3) + " seconds"); + } retries = 0; } catch (SdkClientException sce) { if(retries > 1) { From 953ef621e3ae8ac663d06abe7d77df4be1120149 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Tue, 24 Mar 2020 18:17:04 -0400 Subject: [PATCH 151/157] don't assume optionalParams exists - not true in replace --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 6716cb15327..a470dbcb8e0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -502,10 +502,12 @@ private boolean runAddReplacePhase1(Dataset dataset, return false; } - - if(optionalFileParams.hasCheckSum()) { - newCheckSum = optionalFileParams.getCheckSum(); + if(optionalFileParams != null) { + if(optionalFileParams.hasCheckSum()) { + newCheckSum = optionalFileParams.getCheckSum(); + } } + msgt("step_030_createNewFilesViaIngest"); if (!this.step_030_createNewFilesViaIngest()){ return false; From 9f76eba82cd3e997756a9cae784a09c0c1c0d756 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 25 Mar 2020 14:56:49 -0400 Subject: [PATCH 152/157] Always fix DataFile storageidentifier if needed Not just for write mode --- .../iq/dataverse/dataaccess/S3AccessIO.java | 35 ++++++++++++++----- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index 973fa8ee42a..aa62e605c3c 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -165,7 +165,31 @@ public void open(DataAccessOption... options) throws IOException { if (storageIdentifier == null || "".equals(storageIdentifier)) { throw new FileNotFoundException("Data Access: No local storage identifier defined for this datafile."); } + + + //Fix new DataFiles: DataFiles that have not yet been saved may use this method when they don't have their storageidentifier in the final ://: form + // So we fix it up here. ToDo: refactor so that storageidentifier is generated by the appropriate StorageIO class and is final from the start. + String newStorageIdentifier = null; + if (storageIdentifier.startsWith(this.driverId + "://")) { + if(!storageIdentifier.substring((this.driverId + "://").length()).contains(":")) { + //Driver id but no bucket + if(bucketName!=null) { + newStorageIdentifier=this.driverId + "://" + bucketName + ":" + storageIdentifier.substring((this.driverId + "://").length()); + } else { + throw new IOException("S3AccessIO: DataFile (storage identifier " + storageIdentifier + ") is not associated with a bucket."); + } + } // else we're OK (assumes bucket name in storageidentifier matches the driver's bucketname) + } else { + //No driver id or bucket (presumably - we don't expect cases where there's no driver id but a bucket is listed - could add a check) + newStorageIdentifier= this.driverId + "://" + bucketName + ":" + storageIdentifier; + } + if(newStorageIdentifier != null) { + //Fixup needed: + storageIdentifier = newStorageIdentifier; + dvObject.setStorageIdentifier(newStorageIdentifier); + } + if (isReadAccess) { key = getMainFileKey(); ObjectMetadata objectMetadata = null; @@ -189,14 +213,7 @@ public void open(DataAccessOption... options) throws IOException { } else if (isWriteAccess) { key = dataFile.getOwner().getAuthorityForFileStorage() + "/" + this.getDataFile().getOwner().getIdentifierForFileStorage(); - - if (storageIdentifier.startsWith(this.driverId + "://")) { - key += "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1); - } else { - key += "/" + storageIdentifier; - dvObject.setStorageIdentifier(this.driverId + "://" + bucketName + ":" + storageIdentifier); - } - + key += "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1); } this.setMimeType(dataFile.getContentType()); @@ -784,7 +801,7 @@ String getMainFileKey() throws IOException { if (storageIdentifier.startsWith(this.driverId + "://")) { bucketName = storageIdentifier.substring((this.driverId + "://").length(), storageIdentifier.lastIndexOf(":")); - key = baseKey + "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1); + key = baseKey + "/" + storageIdentifier.substring(storageIdentifier.lastIndexOf(":") + 1); } else { throw new IOException("S3AccessIO: DataFile (storage identifier " + storageIdentifier + ") does not appear to be an S3 object."); } From 445dd89b0febe6c12c963f38df2a6f358917bcaf Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 25 Mar 2020 14:57:18 -0400 Subject: [PATCH 153/157] cleanup - newStorageIdentifier is always set below --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index a470dbcb8e0..53af964fc14 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -952,9 +952,7 @@ private boolean step_020_loadNewFile(String fileName, String fileContentType, St if (storageIdentifier == null) { this.addErrorSevere(getBundleErr("file_upload_failed")); return false; - } else { - newStorageIdentifier = storageIdentifier; - } + } } newFileName = fileName; From d0a0caaae887e6991ea94a2e2e2769d34c824c00 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 25 Mar 2020 15:12:12 -0400 Subject: [PATCH 154/157] add 4th case -only bucket --- .../edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java index aa62e605c3c..b12013b8f8f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/S3AccessIO.java @@ -180,8 +180,13 @@ public void open(DataAccessOption... options) throws IOException { } } // else we're OK (assumes bucket name in storageidentifier matches the driver's bucketname) } else { - //No driver id or bucket (presumably - we don't expect cases where there's no driver id but a bucket is listed - could add a check) - newStorageIdentifier= this.driverId + "://" + bucketName + ":" + storageIdentifier; + if(!storageIdentifier.substring((this.driverId + "://").length()).contains(":")) { + //No driver id or bucket + newStorageIdentifier= this.driverId + "://" + bucketName + ":" + storageIdentifier; + } else { + //Just the bucketname + newStorageIdentifier= this.driverId + "://" + storageIdentifier; + } } if(newStorageIdentifier != null) { //Fixup needed: From 8a26934b9317dfc77dee795fc7eb52bdf4515277 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 25 Mar 2020 17:22:23 -0400 Subject: [PATCH 155/157] avoid null error when select option is removed e.g. for replace file where only one file upload is allowed and the select button goes away after one upload --- src/main/webapp/resources/js/fileupload.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/webapp/resources/js/fileupload.js b/src/main/webapp/resources/js/fileupload.js index c47ce524dc7..126d5d47c87 100644 --- a/src/main/webapp/resources/js/fileupload.js +++ b/src/main/webapp/resources/js/fileupload.js @@ -24,12 +24,14 @@ function setupDirectUpload(enabled, theDatasetId) { $('.ui-fileupload-cancel').hide(); //Catch files entered via upload dialog box. Since this 'select' widget is replaced by PF, we need to add a listener again when it is replaced var fileInput=document.getElementById('datasetForm:fileUpload_input'); - fileInput.addEventListener('change', function(event) { + if(fileInput !==null) { + fileInput.addEventListener('change', function(event) { fileList=[]; for(var i=0;i Date: Wed, 25 Mar 2020 17:23:10 -0400 Subject: [PATCH 156/157] propagate checksum --- .../datasetutility/FileReplacePageHelper.java | 17 ++++++++++++++--- .../datasetutility/OptionalFileParams.java | 4 ++++ 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java index 94f247e6419..24ba8b663bc 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/FileReplacePageHelper.java @@ -94,9 +94,10 @@ public boolean resetReplaceFileHelper(){ /** * Handle native file replace + * @param checkSum * @param event */ - public boolean handleNativeFileUpload(InputStream inputStream, String fullStorageId, String fileName, String fileContentType) { + public boolean handleNativeFileUpload(InputStream inputStream, String fullStorageId, String fileName, String fileContentType, String checkSum) { phase1Success = false; @@ -111,7 +112,17 @@ public boolean handleNativeFileUpload(InputStream inputStream, String fullStorag if (fileContentType == null){ throw new NullPointerException("fileContentType cannot be null"); } - + + OptionalFileParams ofp = null; + if(checkSum != null) { + try { + ofp = new OptionalFileParams(null); + } catch (DataFileTagException e) { + //Shouldn't happen with null input + e.printStackTrace(); + } + ofp.setCheckSum(checkSum); + } // Run 1st phase of replace // replaceFileHelper.runReplaceFromUI_Phase1(fileToReplace.getId(), @@ -119,7 +130,7 @@ public boolean handleNativeFileUpload(InputStream inputStream, String fullStorag fileContentType, inputStream, fullStorageId, - null + ofp ); // Did it work? diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index 11787d25a7e..e48d96e355d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -217,6 +217,10 @@ public String getMimeType() { return mimeType; } + public void setCheckSum(String checkSum) { + this.checkSum = checkSum; + } + public boolean hasCheckSum() { return ((checkSum!=null)&&(!checkSum.isEmpty())); } From bc6b07495f3c44b2819bda3d076ee053888b2c44 Mon Sep 17 00:00:00 2001 From: qqmyers Date: Wed, 25 Mar 2020 17:23:36 -0400 Subject: [PATCH 157/157] convert location to storageId and pass checksum --- .../iq/dataverse/EditDatafilesPage.java | 30 ++++++++++++------- .../iq/dataverse/dataaccess/DataAccess.java | 5 ++++ 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 71b74ddc7ae..8fe7651fe85 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1873,7 +1873,8 @@ private void handleReplaceFileUpload(FacesEvent event, InputStream inputStream, if (fileReplacePageHelper.handleNativeFileUpload(inputStream,null, fileName, - contentType + contentType, + null )){ saveEnabled = true; @@ -1932,11 +1933,13 @@ the uploadFinished() method, triggered next, after the upload event private void handleReplaceFileUpload(String fullStorageLocation, String fileName, - String contentType){ + String contentType, + String checkSum){ fileReplacePageHelper.resetReplaceFileHelper(); saveEnabled = false; - if (fileReplacePageHelper.handleNativeFileUpload(null, fullStorageLocation, fileName, contentType)){ + String storageIdentifier = DataAccess.getStorarageIdFromLocation(fullStorageLocation); + if (fileReplacePageHelper.handleNativeFileUpload(null, storageIdentifier, fileName, contentType, checkSum)){ saveEnabled = true; /** @@ -2069,6 +2072,16 @@ public void handleExternalUpload() { //get file size long fileSize = sio.getSize(); + if(StringUtils.isEmpty(contentType)) { + contentType = FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT; + } + + if(DataFile.ChecksumType.fromString(checksumType) != DataFile.ChecksumType.MD5 ) { + String warningMessage = "Non-MD5 checksums not yet supported in external uploads"; + localWarningMessage = warningMessage; + //ToDo - methods like handleReplaceFileUpload and classes like OptionalFileParams will need to track the algorithm in addition to the value to enable this + } + /* ---------------------------- Check file size - Max size NOT specified in db: default is unlimited @@ -2083,7 +2096,7 @@ public void handleExternalUpload() { // Is this a FileReplaceOperation? If so, then diverge! // ----------------------------------------------------------- if (this.isFileReplaceOperation()){ - this.handleReplaceFileUpload(storageLocation, fileName, FileUtil.MIME_TYPE_UNDETERMINED_DEFAULT); + this.handleReplaceFileUpload(storageLocation, fileName, contentType, checksumValue); this.setFileMetadataSelectedForTagsPopup(fileReplacePageHelper.getNewFileMetadatasBeforeSave().get(0)); return; } @@ -2099,13 +2112,8 @@ public void handleExternalUpload() { // for example, multiple files can be extracted from an uncompressed // zip file. //datafiles = ingestService.createDataFiles(workingVersion, dropBoxStream, fileName, "application/octet-stream"); - if(StringUtils.isEmpty(contentType)) { - contentType = "application/octet-stream"; - } - if(DataFile.ChecksumType.fromString(checksumType) != DataFile.ChecksumType.MD5 ) { - String warningMessage = "Non-MD5 checksums not yet supported in external uploads"; - localWarningMessage = warningMessage; - } + + datafiles = FileUtil.createDataFiles(workingVersion, null, fileName, contentType, fullStorageIdentifier, checksumValue, systemConfig); } catch (IOException ex) { logger.log(Level.SEVERE, "Error during ingest of file {0}", new Object[]{fileName}); diff --git a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java index db87b1751c6..0cf9883b240 100644 --- a/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java +++ b/src/main/java/edu/harvard/iq/dataverse/dataaccess/DataAccess.java @@ -122,6 +122,11 @@ public static String[] getDriverIdAndStorageLocation(String storageLocation) { } public static String getStorarageIdFromLocation(String location) { + if(location.contains("://")) { + //It's a full location with a driverId, so strip and reapply the driver id + //NOte that this will strip the bucketname out (which s3 uses) but the S3IOStorage class knows to look at re-insert it + return location.substring(0,location.indexOf("://") +3) + location.substring(location.lastIndexOf('/')+1); + } return location.substring(location.lastIndexOf('/')+1); }