Skip to content

Commit

Permalink
Merge remote-tracking branch 'origin/main' into rpb-225-fetchAndTrans…
Browse files Browse the repository at this point in the history
…formHebis
  • Loading branch information
TobiasNx committed Jan 17, 2025
2 parents 9fca285 + 08f8792 commit dcfa0c3
Show file tree
Hide file tree
Showing 28 changed files with 251,707 additions and 247,152 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/build.yml
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ on:
push
jobs:
build:
runs-on: ubuntu-latest
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@v2
- name: Set up JDK 1.8
Expand Down
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -24,3 +24,5 @@ RPB-Export_HBZ_ZSS.txt
conf/RPBEXP/*.ZIP
conf/strapi-export.tar.gz
nohup.out*
conf/*.ndjson
conf/*.zip
49 changes: 37 additions & 12 deletions app/controllers/nwbib/Application.java
Original file line number Diff line number Diff line change
Expand Up @@ -259,7 +259,8 @@ public static Promise<Result> search(final String q, final String person,
response().setHeader("Pragma", "no-cache");
response().setHeader("Expires", "0");
}
String cacheId = String.format("%s-%s", uuid, request().uri());
String cacheId = request().queryString().isEmpty() ? request().uri()
: String.format("%s-%s", uuid, request().uri());
@SuppressWarnings("unchecked")
Promise<Result> cachedResult = (Promise<Result>) Cache.get(cacheId);
if (cachedResult != null)
Expand Down Expand Up @@ -907,7 +908,7 @@ public static Promise<Result> showStars(String format, String ids) {
}

public static Promise<Result> showSw(String rpbId) {
String strapiUrl = "https://rpb-cms.lobid.org/admin/content-manager/collectionType/"
String strapiUrl = "https://rpb-cms.lobid.org/admin/content-manager/collection-types/"
+ "api::rpb-authority.rpb-authority?filters[$and][0][rpbId][$eq]=";
return Promise.pure(seeOther(strapiUrl + rpbId));
}
Expand Down Expand Up @@ -950,19 +951,43 @@ private static List<String> starredIds() {
}

public static Promise<Result> put(String id, String secret) throws FileNotFoundException, RecognitionException, IOException {
File input = new File("conf/output/test-output-strapi.json");
File output = new File("conf/output/test-output-0.json");
Files.write(Paths.get(input.getAbsolutePath()), request().body().asJson().toString().getBytes(Charsets.UTF_8));
ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
boolean authorized = !secret.trim().isEmpty() && secret.equals(CONFIG.getString("secret"));
if (authorized) {
String url = "http://weywot3:9200/resources-rpb-test/resource/"
+ URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8");
WSRequest request = WS.url(url).setHeader("Content-Type", "application/json");
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
return transformAndIndex(id, request().body().asJson());
} else {
return Promise.pure(unauthorized(secret));
}
}

public static Promise<Result> delete(String id, String secret) throws FileNotFoundException, RecognitionException, IOException {
boolean authorized = !secret.trim().isEmpty() && secret.equals(CONFIG.getString("secret"));
if (authorized) {
return deleteFromIndex(id);
} else {
return Promise.pure(unauthorized());
return Promise.pure(unauthorized(secret));
}
}

private static Promise<Result> deleteFromIndex(String id) throws UnsupportedEncodingException {
Cache.remove(String.format("/%s", id));
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
return request.delete().map(response -> status(response.getStatus(), response.getBody()));
}

private static Promise<Result> transformAndIndex(String id, JsonNode jsonBody)
throws IOException, FileNotFoundException, RecognitionException, UnsupportedEncodingException {
File input = new File("conf/output/test-output-strapi.json");
File output = new File("conf/output/test-output-0.json");
Files.write(Paths.get(input.getAbsolutePath()), jsonBody.toString().getBytes(Charsets.UTF_8));
ETL.main(new String[] {"conf/rpb-test-titel-to-lobid.flux"});
String result = Files.readAllLines(Paths.get(output.getAbsolutePath())).stream().collect(Collectors.joining("\n"));
Cache.remove(String.format("/%s", id));
WSRequest request = WS.url(elasticsearchUrl(id)).setHeader("Content-Type", "application/json");
return request.put(result).map(response -> status(response.getStatus(), response.getBody()));
}

private static String elasticsearchUrl(String id) throws UnsupportedEncodingException {
return "http://weywot3:9200/resources-rpb-test/resource/"
+ URLEncoder.encode("https://lobid.org/resources/" + id, "UTF-8");
}
}
18 changes: 18 additions & 0 deletions app/controllers/nwbib/Lobid.java
Original file line number Diff line number Diff line change
Expand Up @@ -678,6 +678,24 @@ private static String locationPolygon(String location) {
return location.contains("|") ? location.split("\\|")[1] : location;
}

/**
* @param doc The result JSON doc
* @return A mapping of item IDs (URIs) to item details (JSON strings)
*/
public static Map<String, String> itemDetails(String doc) {
JsonNode items = Json.parse(doc).findValue("hasItem");
Map<String, String> result = new HashMap<>();
if (items != null && (items.isArray() || items.isTextual())) {
Iterator<JsonNode> elements =
items.isArray() ? items.elements() : Arrays.asList(items).iterator();
while (elements.hasNext()) {
JsonNode nextItem = elements.next();
result.put(nextItem.get("id").asText(), nextItem.toString());
}
}
return result;
}

/**
* @param doc The result JSON doc
* @return A mapping of ISILs to item URIs
Expand Down
5 changes: 4 additions & 1 deletion app/rpb/ETL.java
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.io.File;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
Expand Down Expand Up @@ -47,7 +48,9 @@ public static void main(final String[] args) throws FileNotFoundException, Recog
}
vars.put(matcher.group(1), matcher.group(2));
}
FluxCompiler.compile(ResourceUtil.getStream(fluxFile), vars).start();
try (InputStream stream = ResourceUtil.getStream(fluxFile)) {
FluxCompiler.compile(stream, vars).start();
}
}
}
}
8 changes: 6 additions & 2 deletions app/views/TableRow.java
Original file line number Diff line number Diff line change
Expand Up @@ -107,16 +107,18 @@ public String process(JsonNode doc, String property, String param,
private String label(JsonNode doc, String value, List<String> properties) {
List<String> results = new ArrayList<>();
List<String> resultValues = labelsFor(doc, value, properties);
JsonNode labelNode = doc.get(properties.get(0)).iterator().next().get("label");
for (int i = 0; i < resultValues.size(); i++) {
String currentValue = resultValues.get(i);
String[] refAndLabel =
refAndLabel(properties.get(i), currentValue, Optional.empty());
String label = labelNode != null ? labelNode.textValue() : refAndLabel[1];
String result =
properties.get(i).equals("numbering") || value.equals("--")
? currentValue
: String.format(
"<a title=\"Titeldetails anzeigen\" href=\"%s\">%s</a>",
refAndLabel[0], refAndLabel[1]);
refAndLabel[0], label);
results.add(result.replace("Band", "").trim());
}
return results.stream().collect(Collectors.joining(", Band "));
Expand Down Expand Up @@ -215,7 +217,9 @@ String[] refAndLabel(String property, String value,
if ((property.equals("containedIn") || property.equals("hasPart")
|| property.equals("isPartOf") || property.equals("hasSuperordinate")
|| property.equals("bibliographicCitation")) && value.contains("lobid.org")) {
return new String[] { value, Lobid.resourceLabel(value) };
return new String[] { value.matches(".*?[as]\\d+.*|.*?\\d{3}[a-z]\\d+.*") // rpbId
? value.replace("https://lobid.org/resources/", "/")
: value, Lobid.resourceLabel(value) };
}
String label =
labels.isPresent() && labels.get().size() > 0 ? labels.get().get(0)
Expand Down
49 changes: 46 additions & 3 deletions app/views/details.scala.html
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,10 @@
@import play.mvc.Controller.session
@import play.cache.Cache

@idAndLabelLink(e: JsValue) = {
<a href='@((e\"id").asOpt[String].getOrElse("--"))'>@((e\"label").asOpt[String].getOrElse("--"))</a>
}

@pageLink(last: Seq[String], key: String, label: String, tip: String) = {
@defining((last.indexOf(session(key))-1,last.indexOf(session(key))+1)) { case (prev,next) =>
<li class="@if(session(key).isEmpty){disabled}">
Expand Down Expand Up @@ -66,9 +70,9 @@
*@
@tags.search_form(q)
<div class="row" id="search-results">
@defining(Lobid.items(doc.toString)) { items =>
@defining(!(doc\\"containedIn").isEmpty && (doc\"type").toString.contains("Article")){superordination =>
<div class="col-md-12">
@defining((Lobid.items(doc.toString), Lobid.itemDetails(doc.toString))) { case (items, itemDetails) =>
@defining(items.isEmpty && !(doc\\"containedIn").isEmpty && (doc\"type").toString.contains("Article")){superordination =>
<div class="col-md-@if(items.isEmpty && !doc.toString.contains("fulltextOnline") && !superordination){12} else {8}">
<dl>
@defining((doc\"rpbId").asOpt[String].getOrElse(q)){ id =>
<dt>@tags.star_button(id) Titeldetails:
Expand All @@ -80,6 +84,45 @@
}
</dl>
</div>
@if(!items.isEmpty || doc.toString.contains("fulltextOnline") || superordination){
<div class="col-md-4" id="holdings">
<dl>
@if(doc.toString.contains("fulltextOnline")){
<dt>Online-Ressource:</dt>
<dd>
<table class="table table-striped table-condensed">
<tr>
<th style="width: 15%"></th>
<th style="width: 85%"></th>
</tr>
@result_field("Link", "fulltextOnline", doc, TableRow.LINKS)
</table>
</dd>
}
@if(superordination){
<dt>Bestandsangaben:</dt>
<dd>
<table class="table table-striped table-condensed">
<tr>
<th style="width: 40%"></th>
<th style="width: 60%"></th>
</tr>
<tr>
<td>Zum Bestand siehe:</td>
<td>@idAndLabelLink((doc\"containedIn").as[Seq[JsValue]].head)</td>
</tr>
</table>
</dd>
}
@if(!superordination && !items.isEmpty && !doc.toString.contains("fulltextOnline")){
@defining(items.size){num =>
<dt>Bestand in @num Bibliothek@if(num>1){en}:</dt>
}
<dd>@tags.items_map(items, itemDetails)</dd>
}
</dl>
</div>
}
}
}
</div>
Expand Down
8 changes: 6 additions & 2 deletions app/views/index.scala.html
Original file line number Diff line number Diff line change
Expand Up @@ -38,8 +38,8 @@
<div class="col-md-12 intro">
<p class="lead">
Die Rheinland-Pfälzische Bibliographie (RPB) ist eine regionale Literaturdokumentation des Landes Rheinland-Pfalz, die seit 1991 im Auftrag des damaligen
Ministeriums für Wissenschaft und Weiterbildung erstellt wird. Sie erfasst Bücher, Aufsätze aus Zeitschriften, Jahrbüchern, Tages- und Wochenzeitungen sowie
Einzelbeiträge aus Sammelwerken.
Ministeriums für Wissenschaft und Weiterbildung erstellt wird. Sie erfasst Literaturnachweise von Büchern, Aufsätzen aus Zeitschriften, Jahrbüchern, Tages-
und Wochenzeitungen sowie Einzelbeiträgen aus Sammelwerken.
</p>
<p>
Die Bibliographie wird von Mitarbeitern des Landesbibliothekszentrums an den Standorten Speyer und Koblenz sowie den wissenschaftlichen Stadtbibliotheken in
Expand All @@ -48,6 +48,10 @@
erfasst. Die Rheinische Landesbibliothek des lbz führt das Titelmaterial redaktionell zusammen und betreut mit dem hbz die für die Herstellung der Bibliographie
erforderliche Datenverarbeitung.
</p>
<p>
Um an die Artikel oder Bücher selbst heranzukommen, empfiehlt es sich, diese über die eigene Bibliothek vor Ort zu beschaffen (z.B. als Buchfernleihe oder als
Aufsatzbestellung über die Fernleihe).
</p>
</div>
</div>
@*
Expand Down
2 changes: 1 addition & 1 deletion app/views/main.scala.html
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ <h1>Die Rheinland-Pfälzische Bibliographie<br/><small>Ein Entwicklungsprojekt v
<div class="panel panel-default nwbib-footer">
<div class="panel-body">
@defining(controllers.nwbib.Lobid.getTotalHits("").get(Lobid.API_TIMEOUT)){ hits =>
<img class="media-object pull-left nrw-wappen" src="@controllers.routes.Assets.at("images/wappen.png")" alt="NRW-Wappen">
<img class="media-object pull-left nrw-wappen" src="@controllers.routes.Assets.at("images/wappen.png")" alt="RLP-Wappen">
<img class="media-object pull-right hbz-logo" src="@controllers.routes.Assets.at("images/hbz.png")" alt="hbz-Logo">
Die RPB enthält aktuell @hits Literaturnachweise | Ein Entwicklungsprojekt von <a href="https://lbz.rlp.de/">lbz</a> &amp; <a href="http://www.hbz-nrw.de/">hbz</a>
}
Expand Down
134 changes: 134 additions & 0 deletions app/views/tags/items_map.scala.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
@* Copyright 2014-2024 Fabian Steeg, hbz. Licensed under the GPLv2 *@

@(items: Map[String,List[String]], itemDetails: Map[String,String])

@import play.api.libs.ws.WS
@import play.api.libs.json.JsArray
@import play.api.libs.json.Json
@import play.api.libs.json.JsValue
@import scala.concurrent._
@import ExecutionContext.Implicits.global
@import scala.concurrent.duration._
@import play.api.Play.current
@import controllers.nwbib._
@import play.api.cache.Cache

@string(value: JsValue) = { @value.asOpt[String].getOrElse("--") }

<link rel="stylesheet" href="@controllers.routes.Assets.at("stylesheets/leaflet.css")" />
<script src="@controllers.routes.Assets.at("javascripts/leaflet.js")"></script>
@*<script type="text/javascript" src="http://maps.stamen.com/js/tile.stamen.js?v1.2.4"></script>*@
<table class="table table-striped table-condensed">
<tr>
<th style="width: 30%"/>
<th style="width: 70%"/>
</tr>
</table>
<p>
<div class="items-map" id="items-map"></div>
@map_credits()
</p>
<table style="display:none;" id="table" class="table table-striped table-condensed">
</table>
@if(items.size > 1){<a href="#" style="text-decoration-line: underline; text-decoration-style: dotted;" onclick="$('#table').show(); $(this).hide()">Details zum Bestand als Liste anzeigen</a>}
<script>
var layer = L.tileLayer('https://lobid.org/tiles/{z}/{x}/{y}.png');
var rlp = new L.LatLng(49.7, 7.4)
var map = new L.Map("items-map", {
center: rlp,
zoom: 7,
maxZoom: 17,
scrollWheelZoom: true,
attributionControl: false,
zoomControl: false
});
new L.Control.Zoom({ position: 'bottomleft' }).addTo(map);
var markerCount = 0;
var tableDetails = '';
var allTableDetails = '';
var markers = {};
@markers(items)
$('#table').append(allTableDetails);
map.addLayer(layer);
</script>

@markers(items: Map[String,List[String]]) = {
@for((key,i) <- items.keySet.toList.sortWith((k1:String,k2:String)=>Lobid.compareIsil(k1,k2)).zipWithIndex;
owner = Application.CONFIG.getString("orgs.api")+"/"+key;
json = Json.parse(Lobid.cachedJsonCall(owner).toString);
ownerUrl = if((json\\"url").isEmpty) owner else (json\\"url")(0).as[String];
ownerName = if((json\\"name").isEmpty) "" else (json\\"name")(0).as[String]) {
@for((itemId,i) <- items(key).zipWithIndex;
shortItemId = itemId.substring(itemId.lastIndexOf(":") + 1, itemId.lastIndexOf("#"));
itemJson = Json.parse(itemDetails(itemId));
owners = (itemJson\\"heldBy");
if(!owners.isEmpty);
signatures = (itemJson\\"callNumber");
signature = if(signatures.isEmpty) "" else signatures(0).as[String];
urls = (itemJson\\"electronicLocator");
url = if(urls.isEmpty) "" else urls(0).as[String];
seeAlso = (itemJson\\"seeAlso");
opacLink = if(seeAlso.isEmpty) null else seeAlso(0).as[Seq[JsValue]].map(_.as[String])) {
var details = '<tr><td><i>Bibliothek:</i></td><td>'+
'<i>@if(!ownerName.isEmpty){<a href="@ownerUrl">@ownerName</a>}else{&lt;Keine Angabe&gt;}</i></td></tr>'
@if(opacLink!=null){+'<tr><td>Verfügbarkeit:</td>'+'<td><a href="@opacLink">Lokalen Katalog abfragen</a></td></tr>'};
tableDetails = details;
if(allTableDetails=='')
allTableDetails += tableDetails;
else
allTableDetails += '<tr><th style="width: 30%"/><th style="width: 70%"/></tr>' + tableDetails;
var sig = '<tr><td>@if(!signature.isEmpty){Signatur:}@if(!urls.isEmpty){Elektronische Ressource:}</td>'+
'<td>@if(!url.isEmpty){<a href="@url">@shortItemId</a>} else {@if(!signature.isEmpty){@signature}}</td></tr>';
@for(noteSeq <- (itemJson\"note").asOpt[Seq[String]]; note = noteSeq(0)) {sig += '<tr><td>Notiz:</td><td>' + '@note' + '</td></tr>';}
tableDetails += sig;
allTableDetails += sig;
}
@if(
!(json \\ "lon").isEmpty && (json \\ "lon").head.asOpt[String].isDefined &&
!(json \\ "lat").isEmpty && (json \\ "lat").head.asOpt[String].isDefined) {
@defining(i) { id =>
var lat = @string((json \\ "lat").head)
var lon = @string((json \\ "lon").head)
var latlng@(id) = L.latLng(lat, lon);
var icon@(id) = L.icon({
iconUrl: '@controllers.routes.Assets.at("javascripts/images/marker-icon-red.png")',
iconSize: [20, 30],
iconAnchor: [10, 30],
popupAnchor: [0, -30]
});
var marker@(id) = L.marker([lat, lon],{
title: "@ownerName (@key)",
icon: icon@(id)
});
markers[latlng@(id).toString()] = marker@(id);
bindPopup@(id)(tableDetails);
marker@(id).on('click', function(e) {
zoomDetails@(id)();
});
marker@(id).on('popupclose', function(e) {
map.setView(rlp, 7);
});
marker@(id).addTo(map);
markerCount++;
if(@items.size == 1){
bindPopup@(id)(allTableDetails);
zoomDetails@(id)();
}
function zoomDetails@(id)() {
map.setView(latlng@(id), 17);
marker@(id).openPopup();
}
function bindPopup@(id)(content) {
marker@(id).bindPopup(
'<table class="table-striped table-condensed">'+
content+
'</table>',
{
keepInView: true,
maxWidth: 250
});
}
}
}
}
}
Loading

0 comments on commit dcfa0c3

Please sign in to comment.