Merge pull request #268 from europeana/EA-3950_improve_depiction_gene…

…ration

improvements for generation and logging of depictions #EA-3950

set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/DepictionGenerationReport.java

@@ -0,0 +1,35 @@
+package eu.europeana.api.set.integration.migration;
+
+public class DepictionGenerationReport {
+  int skipped = 0;
+  int notGenerated = 0;
+  int generated = 0;
+  int getSkipped() {
+    return skipped;
+  }
+  void setSkipped(int skipped) {
+    this.skipped = skipped;
+  }
+  void increaseSkipped() {
+    this.skipped++;
+  }
+  int getNotGenerated() {
+    return notGenerated;
+  }
+  void setNotGenerated(int notGenerated) {
+    this.notGenerated = notGenerated;
+  }
+  void increaseNotGenerated() {
+    this.notGenerated++;
+  }
+
+  int getGenerated() {
+    return generated;
+  }
+  void setGenerated(int generated) {
+    this.generated = generated;
+  }
+  void increaseGenerated() {
+    this.generated++;
+  }
+}

set-integration-testing/src/test/java/eu/europeana/api/set/integration/migration/GalleryDepictionMigration.java

@@ -4,6 +4,7 @@
 import java.util.List;
 import javax.annotation.Resource;
 import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
 import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.security.core.Authentication;
 import org.springframework.test.context.DynamicPropertyRegistry;
@@ -49,16 +50,18 @@ public static void initTokens() {
 
   @DynamicPropertySource
   static void setProperties(DynamicPropertyRegistry registry) {
-    //  registry.add("mongodb.set.connectionUrl", MONGO_CONTAINER::getConnectionUrl);
-      registry.add("mongodb.set.connectionUrl",  () -> "mongodb://127.0.0.1:27017/set_test"); 
+    registry.add("mongodb.set.connectionUrl",  () -> "mongodb://127.0.0.1:27017/set_test");
+    //registry.add("mongodb.set.connectionUrl",  () -> "");
+    //registry.add("mongodb.set.truststore", () -> "");
+    //registry.add("mongodb.set.truststorepass", () -> "");
   }
 
   /*
    * Generate isShownBy field for all sets in the (local) db
    */
-  //@Test
+  @Test
   public void generateGalleriesWithDepiction() throws Exception {
-    createTestUserSet(USER_SET_REGULAR, regularUserToken);
+//    createTestUserSet(USER_SET_REGULAR, regularUserToken);
 
     // create object in database
     UserSetQueryBuilder queryBuilder = new UserSetQueryBuilder();
@@ -72,12 +75,13 @@ public void generateGalleriesWithDepiction() throws Exception {
         queryBuilder.buildUserSetQuery("type:Collection", null, sort, page, pageSize, getConfiguration());
     final ArrayList<LdProfiles> profiles = new ArrayList<LdProfiles>();
     profiles.add(LdProfiles.STANDARD);
+    DepictionGenerationReport report = new DepictionGenerationReport();
 
     ResultSet<? extends UserSet> results = null;
     do {
       results =
           getUserSetService().search(searchQuery, null, profiles, adminAuth);
-      generateDepictions(results.getResults());
+      generateDepictions(results.getResults(), report);
 
       //move to next page
       page++;
@@ -89,34 +93,49 @@ public void generateGalleriesWithDepiction() throws Exception {
 
   }
 
-  private void generateDepictions(List<? extends UserSet> results) {
+  private void generateDepictions(List<? extends UserSet> results, DepictionGenerationReport report) {
+
     for (UserSet userSet : results) {
+      if(userSet.isOpenSet() || userSet.isBookmarksFolder() || userSet.isEntityBestItemsSet()) {
+        //bookmarks and entity best items sets is redundant, but we keep it for future
+        //open/dynamic sets must not be migrated
+        report.increaseSkipped();
+        continue;
+      }
+      if(userSet.getIsShownBy() != null) {
+        report.increaseSkipped();
+        continue;
+      }
+
       final WebResource isShownBy = generateGalleryDepiction(userSet);
       //do not update set if the depiction cannot be generated
       if(isShownBy != null) {
         userSet.setIsShownBy(isShownBy);
         userSet.setCollectionType(WebUserSetFields.TYPE_GALLERY);
         mongoPersistance.store(userSet);
+        report.increaseGenerated();
+      } else {
+        report.increaseNotGenerated();
       }
     }
+    System.out.println("Generated depictions: " + report.getGenerated());
+    System.out.println("Skipped sets: " + report.getSkipped());
+    System.out.println("Not generated: " + report.getNotGenerated());  
   }
 
   private WebResource generateGalleryDepiction(UserSet userSet){
-    if(userSet.isOpenSet() || userSet.isBookmarksFolder() || userSet.isEntityBestItemsSet()) {
-      //bookmarks and entity best items sets is redundant, but we keep it for future
-      //open/dynamic sets must not be migrated
-      return null;
-    }
-
+
     try {
       return getUserSetService().generateDepiction(userSet);
     } catch (SearchApiClientException e) {
       //work with best user effort
-      System.out.println(e.getMessage());
-      e.printStackTrace();
+      System.out.println("Cannot generate depiciton for set: "+ userSet.getIdentifier() + ", " + e.getMessage());
+      //e.printStackTrace();
       return null;
     }
   }
+
+
 
   private boolean hasNext(final int pageSize, ResultSet<? extends UserSet> results) {
     return results.getResultSize() < pageSize;

set-search-api-client/src/main/java/eu/europeana/set/search/service/impl/SearchApiClientImpl.java

@@ -223,7 +223,7 @@ public void fillDepiction(String searchApiFullUrl, String itemId, BaseWebResourc
 
 
   private String getResourceId(String thumbnailUrl) throws SearchApiClientException {
-    final String queryString = StringUtils.substringAfter(thumbnailUrl, '?');
+    final String queryString = StringUtils.substringAfter(thumbnailUrl, "?");
     List<NameValuePair> params = URLEncodedUtils.parse(queryString, StandardCharsets.UTF_8);
     for (NameValuePair param : params) {
       if ("uri".equals(param.getName())) {

set-web/src/main/java/eu/europeana/set/web/service/impl/UserSetServiceImpl.java

@@ -956,9 +956,13 @@ private boolean isPublishingPrevented(PersistentUserSet userSet) {
 
   @Override
   public WebResource generateDepiction(UserSet userSet) throws SearchApiClientException {
+    if(userSet.getItems() == null || userSet.getItems().isEmpty()) {
+      return null;
+    }
+
     String itemId = userSet.getItems().get(0);
     String url = SearchApiUtils.getInstance().buildSearchApiUrlForItem(getConfiguration().getSearchApiUrl(),
-        itemId, getConfiguration().getSearchApiKey(), "minimal");
+        itemId, getConfiguration().getSearchApiKey(), getConfiguration().getSearchApiProfileForItemDescriptions());
 
     WebResource depiction = new WebResource();
     getSearchApiClient().fillDepiction(url, itemId, depiction);