-
Notifications
You must be signed in to change notification settings - Fork 20
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #958 from mediathekview/feature/ard_topics
ard: use new experimental topics-urls
- Loading branch information
Showing
12 changed files
with
1,417 additions
and
134 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
40 changes: 40 additions & 0 deletions
40
src/main/java/de/mediathekview/mserver/crawler/ard/PaginationUrlDto.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,40 @@ | ||
| package de.mediathekview.mserver.crawler.ard; | ||
|
|
||
| import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO; | ||
|
|
||
| import java.util.HashSet; | ||
| import java.util.Set; | ||
|
|
||
| public class PaginationUrlDto { | ||
| private final Set<CrawlerUrlDTO> urls = new HashSet<>(); | ||
| private int actualPage; | ||
| private int maxPages; | ||
|
|
||
| public void addUrl(CrawlerUrlDTO url) { | ||
| urls.add(url); | ||
| } | ||
|
|
||
| public void addAll(Set<CrawlerUrlDTO> urls) { | ||
| this.urls.addAll(urls); | ||
| } | ||
|
|
||
| public Set<CrawlerUrlDTO> getUrls() { | ||
| return urls; | ||
| } | ||
|
|
||
| public int getActualPage() { | ||
| return actualPage; | ||
| } | ||
|
|
||
| public int getMaxPages() { | ||
| return maxPages; | ||
| } | ||
|
|
||
| public void setActualPage(int actualPage) { | ||
| this.actualPage = actualPage; | ||
| } | ||
|
|
||
| public void setMaxPages(int maxPages) { | ||
| this.maxPages = maxPages; | ||
| } | ||
| } |
51 changes: 51 additions & 0 deletions
51
src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsDeserializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| package de.mediathekview.mserver.crawler.ard.json; | ||
|
|
||
| import com.google.gson.JsonArray; | ||
| import com.google.gson.JsonDeserializationContext; | ||
| import com.google.gson.JsonDeserializer; | ||
| import com.google.gson.JsonElement; | ||
| import de.mediathekview.mserver.base.utils.JsonUtils; | ||
| import de.mediathekview.mserver.crawler.ard.ArdConstants; | ||
| import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO; | ||
| import java.lang.reflect.Type; | ||
| import java.util.*; | ||
|
|
||
| public class ArdTopicsDeserializer implements JsonDeserializer<Set<CrawlerUrlDTO>> { | ||
| private static final String ELEMENT_WIDGETS = "widgets"; | ||
| private static final String ELEMENT_LINKS = "links"; | ||
| private static final String ELEMENT_SELF = "self"; | ||
|
|
||
| private static final String ATTRIBUTE_ID = "id"; | ||
|
|
||
| @Override | ||
| public Set<CrawlerUrlDTO> deserialize( | ||
| JsonElement jsonElement, Type type, JsonDeserializationContext jsonDeserializationContext) { | ||
| final Set<CrawlerUrlDTO> result = new HashSet<>(); | ||
|
|
||
| if (JsonUtils.hasElements(jsonElement, ELEMENT_WIDGETS)) { | ||
| final JsonArray widgets = jsonElement.getAsJsonObject().getAsJsonArray(ELEMENT_WIDGETS); | ||
| widgets.forEach(widget -> parseWidget(widget.getAsJsonObject()).ifPresent(result::add)); | ||
| } | ||
|
|
||
| return result; | ||
| } | ||
|
|
||
| private Optional<CrawlerUrlDTO> parseWidget(final JsonElement compilation) { | ||
| if (JsonUtils.hasElements(compilation, ELEMENT_LINKS)) { | ||
| final JsonElement selfLink = | ||
| compilation.getAsJsonObject().get(ELEMENT_LINKS).getAsJsonObject().get(ELEMENT_SELF); | ||
| final Optional<String> id = | ||
| JsonUtils.getAttributeAsString(selfLink.getAsJsonObject(), ATTRIBUTE_ID); | ||
| if (id.isPresent()) { | ||
| return Optional.of( | ||
| new CrawlerUrlDTO( | ||
| String.format( | ||
| ArdConstants.TOPICS_COMPILATION_URL, | ||
| id.get(), | ||
| ArdConstants.TOPICS_COMPILATION_PAGE_SIZE))); | ||
| } | ||
| } | ||
|
|
||
| return Optional.empty(); | ||
| } | ||
| } |
91 changes: 91 additions & 0 deletions
91
src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsLetterDeserializer.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,91 @@ | ||
| package de.mediathekview.mserver.crawler.ard.json; | ||
|
|
||
| import com.google.gson.JsonDeserializationContext; | ||
| import com.google.gson.JsonDeserializer; | ||
| import com.google.gson.JsonElement; | ||
| import com.google.gson.JsonObject; | ||
| import de.mediathekview.mserver.base.utils.JsonUtils; | ||
| import de.mediathekview.mserver.crawler.ard.ArdConstants; | ||
| import de.mediathekview.mserver.crawler.ard.PaginationUrlDto; | ||
| import de.mediathekview.mserver.crawler.basic.CrawlerUrlDTO; | ||
|
|
||
| import java.lang.reflect.Type; | ||
| import java.util.HashSet; | ||
| import java.util.Optional; | ||
| import java.util.Set; | ||
|
|
||
| public class ArdTopicsLetterDeserializer implements JsonDeserializer<PaginationUrlDto> { | ||
|
|
||
| private static final String ELEMENT_TEASERS = "teasers"; | ||
| private static final String ELEMENT_LINKS = "links"; | ||
| private static final String ELEMENT_TARGET = "target"; | ||
| private static final String ELEMENT_PAGE_NUMBER = "pageNumber"; | ||
| private static final String ELEMENT_TOTAL_ELEMENTS = "totalElements"; | ||
| private static final String ELEMENT_PAGE_SIZE = "pageSize"; | ||
| private static final String ELEMENT_PAGINATION = "pagination"; | ||
|
|
||
| private static final String ATTRIBUTE_ID = "id"; | ||
|
|
||
| @Override | ||
| public PaginationUrlDto deserialize( | ||
| final JsonElement jsonElement, final Type type, final JsonDeserializationContext context) { | ||
| final PaginationUrlDto results = new PaginationUrlDto(); | ||
|
|
||
| if (!jsonElement.getAsJsonObject().has(ELEMENT_TEASERS) | ||
| || !jsonElement.getAsJsonObject().get(ELEMENT_TEASERS).isJsonArray() | ||
| || jsonElement.getAsJsonObject().getAsJsonArray(ELEMENT_TEASERS).isEmpty()) { | ||
| return results; | ||
| } | ||
|
|
||
| jsonElement.getAsJsonObject().getAsJsonArray(ELEMENT_TEASERS).forEach(teaser -> results.addAll(parseTeaser(teaser.getAsJsonObject()))); | ||
|
|
||
| final JsonElement paginationElement = jsonElement.getAsJsonObject().get(ELEMENT_PAGINATION); | ||
| results.setActualPage(getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_NUMBER)); | ||
| final int totalElements = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_TOTAL_ELEMENTS); | ||
| final int pageSize = getChildElementAsIntOrNullIfNotExist(paginationElement, ELEMENT_PAGE_SIZE); | ||
| int maxPageSize = pageSize == 0 ? 0 : | ||
| (totalElements+pageSize-1)/pageSize; | ||
| results.setMaxPages(maxPageSize); | ||
|
|
||
| return results; | ||
| } | ||
|
|
||
| private int getChildElementAsIntOrNullIfNotExist( | ||
| final JsonElement parentElement, final String childElementName) { | ||
| if (parentElement == null || parentElement.isJsonNull()) { | ||
| return 0; | ||
| } | ||
| return getJsonElementAsIntOrNullIfNotExist( | ||
| parentElement.getAsJsonObject().get(childElementName)); | ||
| } | ||
|
|
||
| private int getJsonElementAsIntOrNullIfNotExist(final JsonElement element) { | ||
| if (element.isJsonNull()) { | ||
| return 0; | ||
| } | ||
| return element.getAsInt(); | ||
| } | ||
|
|
||
| private Set<CrawlerUrlDTO> parseTeaser(final JsonObject teaserObject) { | ||
| final Set<CrawlerUrlDTO> results = new HashSet<>(); | ||
|
|
||
| final Optional<String> id; | ||
|
|
||
| if (JsonUtils.checkTreePath(teaserObject, null, ELEMENT_LINKS, ELEMENT_TARGET)) { | ||
| final JsonObject targetObject = | ||
| teaserObject.get(ELEMENT_LINKS).getAsJsonObject().get(ELEMENT_TARGET).getAsJsonObject(); | ||
| id = JsonUtils.getAttributeAsString(targetObject, ATTRIBUTE_ID); | ||
| } else { | ||
| id = JsonUtils.getAttributeAsString(teaserObject, ATTRIBUTE_ID); | ||
| } | ||
|
|
||
| id.ifPresent( | ||
| nonNullId -> | ||
| results.add( | ||
| new CrawlerUrlDTO( | ||
| String.format( | ||
| ArdConstants.TOPIC_URL, nonNullId, ArdConstants.TOPIC_PAGE_SIZE)))); | ||
|
|
||
| return results; | ||
| } | ||
| } |
91 changes: 0 additions & 91 deletions
91
src/main/java/de/mediathekview/mserver/crawler/ard/json/ArdTopicsOverviewDeserializer.java
This file was deleted.
Oops, something went wrong.
Oops, something went wrong.