Merge remote-tracking branch 'origin/main' into main

README.md

@@ -84,14 +84,14 @@ This attempts to import all data selected with the `PICK` variable to the API en
 To reimport existing entries, these may need to be deleted first, e.g. for `articles/1` to `articles/5`:
 
 ```
-curl --request DELETE http://test-metadaten-nrw.hbz-nrw.de:1339/api/articles/[1-5]
+curl --request DELETE http://test-metadaten-nrw.hbz-nrw.de:1337/api/articles/[1-5]
 ```
 
-After import they are available at e.g. http://test-metadaten-nrw.hbz-nrw.de:1339/api/articles?populate=*
+After import they are available at e.g. http://test-metadaten-nrw.hbz-nrw.de:1337/api/articles?populate=*
 
 Entries using the same path can be filtered, e.g. to get only volumes (`f36_=sbd`):
 
-http://test-metadaten-nrw.hbz-nrw.de:1339/api/independent-works?filters[f36_][$eq]=sbd&populate=*
+http://test-metadaten-nrw.hbz-nrw.de:1337/api/independent-works?filters[f36_][$eq]=sbd&populate=*
 
 ### Import SKOS data into strapi
 

conf/rpb-sw-import.flux

@@ -1,6 +1,7 @@
 default IN_FILE = "test-output-sw.json"; // pass e.g. IN_FILE=output-sw-strapi.ndjson
 default HOST = "localhost"; // pass e.g. HOST=test-metadaten-nrw.hbz-nrw.de
-API_URL = "http://" + HOST + ":1339/api/rpb-authorities";
+default API_TOKEN = ""; // pass e.g. API_TOKEN=e8d...
+API_URL = "http://" + HOST + ":1337/api/rpb-authorities";
 
 FLUX_DIR + "output/" + IN_FILE
 | open-file
@@ -9,7 +10,7 @@ FLUX_DIR + "output/" + IN_FILE
 | stream-to-triples
 | template("{\"${p}\":${o}}") // wrap into 'data' object for strapi
 | log-object("Will POST: ")
-| open-http(url=API_URL, method="POST", contentType="application/json")
+| open-http(url=API_URL, method="POST", contentType="application/json", header="Authorization: Bearer " + API_TOKEN)
 | as-lines
 | log-object("POST Response: ")
 ;

conf/rpb-systematik-import.flux

@@ -1,5 +1,6 @@
 default HOST = "localhost"; // pass e.g. HOST=test-metadaten-nrw.hbz-nrw.de
-API_URL = "http://" + HOST + ":1339/api/" + PATH;
+default API_TOKEN = ""; // pass e.g. API_TOKEN=e8d...
+API_URL = "http://" + HOST + ":1337/api/" + PATH;
 
 "https://raw.githubusercontent.com/acka47/scripts/master/skos2json/" + INPUT
 | open-http
@@ -8,7 +9,7 @@ API_URL = "http://" + HOST + ":1339/api/" + PATH;
 | stream-to-triples
 | template("{\"${p}\":${o}}") // wrap into 'data' object for strapi
 | log-object("Will POST: ")
-| open-http(url=API_URL, method="POST", contentType="application/json")
+| open-http(url=API_URL, method="POST", contentType="application/json", header="Authorization: Bearer " + API_TOKEN)
 | as-lines
 | log-object("POST Response: ")
 ;

conf/rpb-test-titel-import.flux

@@ -1,5 +1,6 @@
 default HOST = "localhost"; // pass e.g. test-metadaten-nrw.hbz-nrw.de
-API_URL = "http://" + HOST + ":1339/api/" + PATH;
+default API_TOKEN = ""; // pass e.g. API_TOKEN=e8d...
+API_URL = "http://" + HOST + ":1337/api/" + PATH;
 
 FIX = "
 unless " + PICK + "
@@ -17,7 +18,7 @@ FLUX_DIR + "output/test-output-strapi.json"
 | stream-to-triples
 | template("{\"${p}\":${o}}") // wrap into 'data' object for strapi
 | log-object("Will POST: ")
-| open-http(url=API_URL, method="POST", contentType="application/json")
+| open-http(url=API_URL, method="POST", contentType="application/json", header="Authorization: Bearer " + API_TOKEN)
 | as-lines
 | log-object("POST Response: ")
 ;

conf/rpb-titel-import.flux

@@ -1,5 +1,6 @@
 default HOST = "localhost"; // pass e.g. test-metadaten-nrw.hbz-nrw.de
-API_URL = "http://" + HOST + ":1339/api/" + PATH;
+default API_TOKEN = ""; // pass e.g. API_TOKEN=e8d...
+API_URL = "http://" + HOST + ":1337/api/" + PATH;
 
 FIX = "
 unless " + PICK + "
@@ -17,7 +18,7 @@ FLUX_DIR + "output/output-strapi.ndjson"
 | stream-to-triples
 | template("{\"${p}\":${o}}") // wrap into 'data' object for strapi
 | log-object("Will POST: ")
-| open-http(url=API_URL, method="POST", contentType="application/json")
+| open-http(url=API_URL, method="POST", contentType="application/json", header="Authorization: Bearer " + API_TOKEN)
 | as-lines
 | log-object("POST Response: ")
 ;

conf/rppd-import.flux

@@ -1,6 +1,7 @@
 default HOST = "localhost"; // pass e.g. HOST=test-metadaten-nrw.hbz-nrw.de
 default IN_FILE = "test-output-rppd.json"; // pass e.g. IN_FILE=output-rppd-strapi.ndjson
-API_URL = "http://" + HOST + ":1339/api/rppds";
+default API_TOKEN = ""; // pass e.g. API_TOKEN=e8d...
+API_URL = "http://" + HOST + ":1337/api/rppds";
 
 FLUX_DIR + "output/" + IN_FILE
 | open-file
@@ -9,7 +10,7 @@ FLUX_DIR + "output/" + IN_FILE
 | stream-to-triples
 | template("{\"${p}\":${o}}") // wrap into 'data' object for strapi
 | log-object("Will POST: ")
-| open-http(url=API_URL, method="POST", contentType="application/json")
+| open-http(url=API_URL, method="POST", contentType="application/json", header="Authorization: Bearer " + API_TOKEN)
 | as-lines
 | log-object("POST Response: ")
 ;

conf/rppd-to-lobid.fix

@@ -289,13 +289,12 @@ copy_field("f1nu[].*.f1nu", "publication[].$append")
 
 # -------
 # #1nv	(Geschlecht) -> 	gender	
-# Kommentar Doku: "mmm" (männlich) oder "www" (weiblich) eintragen, 
 
 set_array("gender[]")
-if any_equal("f1nv", "mmm")
+if any_equal("f1nv", "männlich")
   set_field("gender[].$append.id", "https://d-nb.info/standards/vocab/gnd/gender#male")
   set_field("gender[].$last.label", "Männlich")
-elsif any_equal("f1nv", "www")
+elsif any_equal("f1nv", "weiblich")
   set_field("gender[].$append.id", "https://d-nb.info/standards/vocab/gnd/gender#female")
   set_field("gender[].$last.label", "Weiblich")
 else

conf/rppd-to-strapi.fix

@@ -33,6 +33,7 @@ do once()
 end
 
 call_macro("to_bool", field: "f11_")
+call_macro("to_bool", field: "f12_")
 call_macro("to_bool", field: "f13_")
 call_macro("to_bool", field: "f14_")
 call_macro("to_bool", field: "f15_")
@@ -63,7 +64,8 @@ end
 replace_all("f82b", "(\\d+)n(\\d)", "$1-$2")
 lookup("f82b", "map_gndid_to_new")
 
-replace_all("f1nv", "w+", "www") # z.B. ww -> www
-replace_all("f1nv", "m+", "mmm") # z.B. mmmm -> mmm
+replace_all("f1nv", "w+", "weiblich")
+replace_all("f1nv", "m+", "männlich")
+replace_all("f1nv", "d+", "divers")
 
 vacuum()

transformAndImportFull.sh

@@ -6,28 +6,30 @@ IFS=$'\n\t'
 
 if [ -z "$@" ]; then echo "Pass host, e.g. 'localhost'"; else echo "Will import to '$@'"; fi
 HOST=$1
+# Create token in Strapi admin UI (Settings > API Tokens), set here or in environment
+API_TOKEN=$API_TOKEN
 
 # Notationen
-# curl --request DELETE "http://$HOST:1339/api/rpb-notations/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=$HOST"
-# curl --request DELETE "http://$HOST:1339/api/rpb-spatials/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=$HOST"
+# curl --request DELETE "http://$HOST:1337/api/rpb-notations/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=$HOST API_TOKEN=$API_TOKEN"
+# curl --request DELETE "http://$HOST:1337/api/rpb-spatials/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Personen
-# curl --request DELETE "http://$HOST:1339/api/rppds/[1-5]"
+# curl --request DELETE "http://$HOST:1337/api/rppds/[1-5]"
 sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio.txt OUT_FILE=output-rppd-strapi.ndjson"
-sbt "runMain rpb.ETL conf/rppd-import.flux IN_FILE=output-rppd-strapi.ndjson HOST=$HOST"
+sbt "runMain rpb.ETL conf/rppd-import.flux IN_FILE=output-rppd-strapi.ndjson HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Normdaten
-# curl --request DELETE "http://$HOST:1339/api/rpb-authorities/[1-5]"
+# curl --request DELETE "http://$HOST:1337/api/rpb-authorities/[1-5]"
 sbt "runMain rpb.ETL conf/rpb-sw-to-strapi.flux IN_FILE=RPB-Export_HBZ_SW.txt OUT_FILE=output-sw-strapi.ndjson"
-sbt "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=output-sw-strapi.ndjson HOST=$HOST"
+sbt "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=output-sw-strapi.ndjson HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Titeldaten
 sbt "runMain rpb.ETL conf/rpb-titel-to-strapi.flux"
-# curl --request DELETE "http://$HOST:1339/api/articles/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36_','u') PATH=articles HOST=$HOST"
-# curl --request DELETE "http://$HOST:1339/api/independent-works/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36_','Monografie') PATH=independent-works HOST=$HOST"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36_','Band') PATH=independent-works HOST=$HOST"
-sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36t','MultiVolumeBook') PATH=independent-works HOST=$HOST"
+# curl --request DELETE "http://$HOST:1337/api/articles/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36_','u') PATH=articles HOST=$HOST API_TOKEN=$API_TOKEN"
+# curl --request DELETE "http://$HOST:1337/api/independent-works/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36_','Monografie') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36_','Band') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt "runMain rpb.ETL conf/rpb-titel-import.flux PICK=all_equal('f36t','MultiVolumeBook') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"

transformAndImportTest.sh

@@ -6,28 +6,30 @@ IFS=$'\n\t'
 
 if [ -z "$@" ]; then echo "Pass host, e.g. 'localhost'"; else echo "Will import to '$@'"; fi
 HOST=$1
+# Create token in Strapi admin UI (Settings > API Tokens), set here or in environment
+API_TOKEN=$API_TOKEN
 
 # Notationen
-# curl --request DELETE "http://$HOST:1339/api/rpb-notations/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=$HOST"
-# curl --request DELETE "http://$HOST:1339/api/rpb-spatials/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=$HOST"
+# curl --request DELETE "http://$HOST:1337/api/rpb-notations/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb.ndjson PATH=rpb-notations HOST=$HOST API_TOKEN=$API_TOKEN"
+# curl --request DELETE "http://$HOST:1337/api/rpb-spatials/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-systematik-import.flux INPUT=rpb-spatial.ndjson PATH=rpb-spatials HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Personen
-# curl --request DELETE "http://$HOST:1339/api/rppds/[1-5]"
+# curl --request DELETE "http://$HOST:1337/api/rppds/[1-5]"
 sbt "runMain rpb.ETL conf/rppd-to-strapi.flux IN_FILE=RPB-Export_HBZ_Bio_Test.txt OUT_FILE=test-output-rppd.json"
-sbt "runMain rpb.ETL conf/rppd-import.flux IN_FILE=test-output-rppd.json HOST=$HOST"
+sbt "runMain rpb.ETL conf/rppd-import.flux IN_FILE=test-output-rppd.json HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Normdaten
-# curl --request DELETE "http://$HOST:1339/api/rpb-authorities/[1-5]"
+# curl --request DELETE "http://$HOST:1337/api/rpb-authorities/[1-5]"
 sbt "runMain rpb.ETL conf/rpb-sw-to-strapi.flux IN_FILE=RPB-Export_HBZ_SW_Test.txt OUT_FILE=test-output-sw.json"
-sbt "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=test-output-sw.json HOST=$HOST"
+sbt "runMain rpb.ETL conf/rpb-sw-import.flux IN_FILE=test-output-sw.json HOST=$HOST API_TOKEN=$API_TOKEN"
 
 # Titeldaten
 sbt "runMain rpb.ETL conf/rpb-test-titel-to-strapi.flux"
-# curl --request DELETE "http://$HOST:1339/api/articles/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','u') PATH=articles HOST=$HOST"
-# curl --request DELETE "http://$HOST:1339/api/independent-works/[1-5]"
-sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','Monografie') PATH=independent-works HOST=$HOST"
-sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','Band') PATH=independent-works HOST=$HOST"
-sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36t','MultiVolumeBook') PATH=independent-works HOST=$HOST"
+# curl --request DELETE "http://$HOST:1337/api/articles/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','u') PATH=articles HOST=$HOST API_TOKEN=$API_TOKEN"
+# curl --request DELETE "http://$HOST:1337/api/independent-works/[1-5]"
+sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','Monografie') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36_','Band') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"
+sbt "runMain rpb.ETL conf/rpb-test-titel-import.flux PICK=all_equal('f36t','MultiVolumeBook') PATH=independent-works HOST=$HOST API_TOKEN=$API_TOKEN"