Skip to content

Commit

Permalink
Merge branch 'develop' into feature/linked-sab-classification
Browse files Browse the repository at this point in the history
  • Loading branch information
@niklasl
niklasl committed Dec 9, 2024
2 parents 37172ed + 32fdaf5 commit 9eb2893
Show file tree
Hide file tree
Showing 256 changed files with 2,174,840 additions and 4,621 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ Session.vim
/*.properties
/*.properties-*
!gradle.properties
emm/exampleclient/libris-cache.sqlite3

# Java/Gradle Artifacts
/.gradle
Expand Down
4 changes: 2 additions & 2 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -234,8 +234,8 @@ instead of `rest`.)
To run the frontend, first set up the Libris cataloging client and the
id.kb.se web app (follow the README in each):
* [Libris cataloging](https://github.com/libris/lxlviewer/tree/develop/vue-client)
* [id.kb.se](https://github.com/libris/lxlviewer/tree/develop/nuxt-app)
* [Libris cataloging](https://github.com/libris/lxlviewer/tree/develop/cataloging)
* [id.kb.se](https://github.com/libris/lxlviewer/tree/develop/idkbse)
At this point, you should have the LXLViewer cataloging client running on port 8080
and the id.kb.se app running on port 3000, but they won't work yet.
Expand Down
82 changes: 82 additions & 0 deletions batchimport/integtest/batch15.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
<?xml version="1.0" encoding="UTF-8"?>
<collection xmlns="http://www.loc.gov/MARC21/slim">
<record type="Bibliographic">
<leader>02625cam a22007458i 4500</leader>
<controlfield tag="001">20203738</controlfield>
<controlfield tag="003">SE-LIBR</controlfield>
<controlfield tag="005">20170713141440.0</controlfield>
<controlfield tag="008">170327s2017 sw ||||j |00| 0|swe d</controlfield>
<datafield tag="020" ind1=" " ind2=" ">
<subfield code="a">978-91-502-2231-9</subfield>
<subfield code="q">inbunden</subfield>
</datafield>
<datafield tag="035" ind1=" " ind2=" ">
<subfield code="a">(BOKR)9789150222319_OLD</subfield>
</datafield>
<datafield tag="040" ind1=" " ind2=" ">
<subfield code="a">BOKR</subfield>
</datafield>
<datafield tag="100" ind1="1" ind2=" ">
<subfield code="a">Bently, Peter</subfield>
<subfield code="4">aut</subfield>
</datafield>
<datafield tag="245" ind1="1" ind2="0">
<subfield code="a">ANNAN TITEL</subfield>
</datafield>
<datafield tag="263" ind1=" " ind2=" ">
<subfield code="a">201709</subfield>
</datafield>
<datafield tag="264" ind1=" " ind2="1">
<subfield code="b">Berghs,</subfield>
<subfield code="c">2017</subfield>
</datafield>
<datafield tag="300" ind1=" " ind2=" ">
<subfield code="a">32 sidor :</subfield>
<subfield code="b">illustrationer</subfield>
</datafield>
<datafield tag="336" ind1=" " ind2=" ">
<subfield code="b">txt</subfield>
<subfield code="2">rdacontent</subfield>
</datafield>
<datafield tag="337" ind1=" " ind2=" ">
<subfield code="b">n</subfield>
<subfield code="2">rdamedia</subfield>
</datafield>
<datafield tag="338" ind1=" " ind2=" ">
<subfield code="b">nc</subfield>
<subfield code="2">rdacarrier</subfield>
</datafield>
<datafield tag="599" ind1=" " ind2=" ">
<subfield code="a">Maskinellt genererad post. Ändra kod för fullständighetsnivå (leader/17), annars kommer manuellt gjorda ändringar att försvinna.</subfield>
</datafield>
<datafield tag="700" ind1="1" ind2=" ">
<subfield code="a">Lightfoot, Marta</subfield>
<subfield code="4">ill</subfield>
</datafield>
<datafield tag="700" ind1="1" ind2=" ">
<subfield code="a">Vidén, Eva</subfield>
<subfield code="4">trl</subfield>
</datafield>
</record>
<record type="Holdings">
<leader>XXXXXax a22XXXXX1n 4500</leader>
<controlfield tag="008">170714||0000|||||000||||||000000</controlfield>
<datafield tag="852" ind1="" ind2="">
<subfield code="b">Sods</subfield>
</datafield>
</record>
<record type="Holdings">
<leader>XXXXXax a22XXXXX1n 4500</leader>
<controlfield tag="008">170714||0000|||||000||||||000000</controlfield>
<datafield tag="852" ind1="" ind2="">
<subfield code="b">Li</subfield>
</datafield>
</record>
<record type="Holdings">
<leader>XXXXXax a22XXXXX1n 4500</leader>
<controlfield tag="008">170714||0000|||||000||||||000000</controlfield>
<datafield tag="852" ind1="" ind2="">
<subfield code="b">G</subfield>
</datafield>
</record>
</collection>
76 changes: 37 additions & 39 deletions batchimport/integtest/test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -184,45 +184,6 @@ if (( $rowCount != 3 )) ; then
fail "Expected exactly 3 hold records"
fi

cleanup
######## Electronic/Instance should not match. Batch 6 contains the same record, but work type is Multimedia instead of Text
#java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch0.xml --format=xml --live --changedIn=importtest
#java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch6.xml --format=xml --dupType=ISBNA --live --changedIn=importtest
#rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'bib'")
#if (( $rowCount != 2 )) ; then
# fail "Expected 2 bib records after importing Instance and Electronic"
#fi
#rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'hold'")
#if (( $rowCount != 2 )) ; then
# fail "Expected 2 bib records after importing Instance and Electronic"
#fi

cleanup
######## Electronic/Instance should not match. Batch 7 contains the same record, but instance type is electronic
#java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch0.xml --format=xml --live --changedIn=importtest
#java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch7.xml --format=xml --dupType=ISBNA --live --changedIn=importtest
#rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'bib'")
#if (( $rowCount != 2 )) ; then
# fail "Expected 2 bib records after importing Instance and Electronic"
#fi
#rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'hold'")
#if (( $rowCount != 2 )) ; then
# fail "Expected 2 bib records after importing Instance and Electronic"
#fi

cleanup
######## Electronic/Instance should not match. Batch 9 contains the same record, but instance type is electronic
#java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch0.xml --format=xml --live --changedIn=importtest
#java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch9.xml --format=xml --dupType=ISBNA --live --changedIn=importtest
#rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'bib'")
#if (( $rowCount != 2 )) ; then
# fail "Expected 2 bib records after importing Instance and Electronic"
#fi
#rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'hold'")
#if (( $rowCount != 2 )) ; then
# fail "Expected 2 bib records after importing Instance and Electronic"
#fi

cleanup
######## ISBN $z 10/13 matching. Batch 4 and 8 contain the same data but with varying forms of the same ISBN in 020$z.
java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch4.xml --format=xml --live --changedIn=importtest --changedBy=Utb1
Expand Down Expand Up @@ -298,6 +259,43 @@ if [ "$summary" != "$expect" ]; then
fi


cleanup
# Test the --ignoreNewBib flag. Import the second time without matching any thing. No new bib should be created, and the associated holds should also be discarded.
java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch0.xml --format=xml --live --changedIn=importtest --changedBy=Utb1
java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch15.xml --format=xml --live --changedIn=importtest --changedBy=Utb1 --ignoreNewBib
mainTitle=$($PSQL -qAt whelk_dev <<< "select data from lddb where changedIn = 'importtest' and collection = 'bib'" | jq '.["@graph"]|.[1]|.["hasTitle"]|.[0]|.["mainTitle"]')
expect="\"ANNAN TITEL\""
if [ "$mainTitle" == "$expect" ]; then
fail "Bib should not have been added/replaced/merged, yet the new title is there!"
fi
rowCount=$($PSQL -qAt whelk_dev <<< "select count(*) from lddb where changedIn = 'importtest' and collection = 'hold'")
if (( $rowCount != 1 )) ; then
fail "Expected single hold record, as the other two holdings sat on a bib we're choosing to discard."
fi


#Rerun the simple bib-replace test, but with the --ignoreNewBib flag, to check that it does not interfere. batch1.xml contains the same data but with a changed title for the bib record and another 035$a
java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch0.xml --format=xml --dupType=ISBNA,ISBNZ,ISSNA,ISSNZ,035A --live --changedIn=importtest --changedBy=Utb1
bibResourceId=$($PSQL -qAt whelk_dev <<< "select data from lddb where changedIn = 'importtest' and collection = 'bib'" | jq '.["@graph"]|.[1]|.["@id"]')
java "$SECRET_PROPS" -jar build/libs/batchimport.jar --path=./integtest/batch1.xml --format=xml --dupType=ISBNA,ISBNZ,ISSNA,ISSNZ,035A --live --changedIn=importtest --replaceBib --ignoreNewBib --changedBy=Utb1
newBibResourceId=$($PSQL -qAt whelk_dev <<< "select data from lddb where changedIn = 'importtest' and collection = 'bib'" | jq '.["@graph"]|.[1]|.["@id"]')
if [ $newBibResourceId != $bibResourceId ]; then
fail "Bib-replace altered the ID!"
fi
mainTitle=$($PSQL -qAt whelk_dev <<< "select data from lddb where changedIn = 'importtest' and collection = 'bib'" | jq '.["@graph"]|.[1]|.["hasTitle"]|.[0]|.["mainTitle"]')
expect="\"Polisbilen får INTE ett larm\""
if [ "$mainTitle" != "$expect" ]; then
fail "Data was not replaced!"
fi
systemNumbers=$($PSQL -qAt whelk_dev <<< "select data from lddb where changedIn = 'importtest' and collection = 'bib'" | jq '.["@graph"]|.[0]|.["identifiedBy"]|.[]|.["value"]')
if [[ "$systemNumbers" != *"NEW"* ]]; then
fail "Systemnumber was not added!"
fi
if [[ "$systemNumbers" != *"OLD"* ]]; then
fail "Systemnumber was not retained!"
fi


cleanup
popd
echo $OUTCOME
10 changes: 10 additions & 0 deletions batchimport/src/main/java/whelk/importer/Parameters.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ class Parameters
private boolean verbose = false;
private boolean replaceHold = false;
private boolean replaceBib = false;
private boolean ignoreNewBib = false;
private String changedBy = null;
private String changedIn = null;
private boolean forceUpdate = false;
Expand All @@ -40,6 +41,7 @@ class Parameters
boolean getVerbose() { return verbose; }
boolean getReplaceHold() { return replaceHold; }
boolean getReplaceBib() { return replaceBib; }
boolean getIgnoreNewBib() { return ignoreNewBib; }
String getChangedBy() { return changedBy; }
String getChangedIn() { return changedIn; }
boolean getForceUpdate() { return forceUpdate; }
Expand Down Expand Up @@ -159,6 +161,11 @@ private void printUsage()
System.err.println();
System.err.println("--replaceBib If this flag is set, matching bibliographic records will be replaced.");
System.err.println();
System.err.println("--ignoreNewBib If this flag is set, no _new_ bibliographic records will be created");
System.err.println(" (existing ones may still optionally be replaced/merged).");
System.err.println(" any holding records associated with an incoming bibliographic record that did not match");
System.err.println(" and which cannot be created, because of this flag, will be discarded.");
System.err.println();
System.err.println("--replaceHold If this flag is set, matching holding records will be replaced.");
System.err.println();
System.err.println("--mergeBibUsing Bibliographic records can be \"partially uppated\" (merged) with incoming");
Expand Down Expand Up @@ -314,6 +321,9 @@ private void interpretUnaryParameter(String parameter)
throw new IllegalArgumentException("Encrich/replace are mutually exclusive");
replaceBib = true;
break;
case "--ignoreNewBib":
ignoreNewBib = true;
break;
case "--forceUpdate":
forceUpdate = true;
break;
Expand Down
Loading

0 comments on commit 9eb2893

Please sign in to comment.