|
|\n", perl = TRUE)) +paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] + +# Split paragraph +paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) +purrr::map(paragraphs, ~ if(grepl("^ the European", .x)) { + .x <- paste0("Data were exported from", .x) + } else {.x <- .x}) + +# Rearrange paragraphs: abstract + additional_info +eml$dataset$abstract$para <- c(eml$dataset$abstract$para, paragraphs) + +# Delete `additionalInfo` +eml$dataset$additionalInfo <- NULL +``` + + +# Write EML + +```{r} +eml_path <- here::here("data", project_id, "processed", "eml.xml") +EML::write_eml(eml, eml_path) +``` + + From 9d7d5c0c20d2c3bd7b40b7a1c0512d8588fc89f8 Mon Sep 17 00:00:00 2001 From: Sanne Govaert <44606923+sannegovaert@users.noreply.github.com> Date: Thu, 12 Sep 2024 17:17:04 +0200 Subject: [PATCH 02/19] upload data --- data/2011_RIVIERPRIK/processed/eml.xml | 242 +++++++++++++++++++ data/2011_RIVIERPRIK/raw/eml.xml | 264 +++++++++++++++++++++ data/2012_LEOPOLDKANAAL/raw/eml.xml | 306 +++++++++++++++++++++++++ data/2013_ALBERTKANAAL/raw/eml.xml | 305 ++++++++++++++++++++++++ data/2014_DEMER/raw/eml.xml | 287 +++++++++++++++++++++++ data/2015_DIJLE/processed/eml.xml | 304 ++++++++++++++++++++++++ 6 files changed, 1708 insertions(+) create mode 100644 data/2011_RIVIERPRIK/processed/eml.xml create mode 100644 data/2011_RIVIERPRIK/raw/eml.xml create mode 100644 data/2012_LEOPOLDKANAAL/raw/eml.xml create mode 100644 data/2013_ALBERTKANAAL/raw/eml.xml create mode 100644 data/2014_DEMER/raw/eml.xml create mode 100644 data/2015_DIJLE/processed/eml.xml diff --git a/data/2011_RIVIERPRIK/processed/eml.xml b/data/2011_RIVIERPRIK/processed/eml.xml new file mode 100644 index 0000000..6395ff3 --- /dev/null +++ b/data/2011_RIVIERPRIK/processed/eml.xml @@ -0,0 +1,242 @@ + +The study was commissioned by the Vlaamse Waterweg NV. Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>This dataset was collected using infrastructure provided by VLIZ and INBO funded by the Research Foundation - Flanders (FWO) as part of the Belgian contribution to LifeWatch. The study was commissioned by the Agency of Nature and Forest (ANB). Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>This dataset was collected using infrastructure provided by VLIZ and INBO funded by the Research Foundation - Flanders (FWO) as part of the Belgian contribution to LifeWatch. The study was commissioned by the Vlaamse Waterweg NV. Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>This dataset was collected using infrastructure provided by VLIZ and INBO funded by the Research Foundation - Flanders (FWO) as part of the Belgian contribution to LifeWatch. The study was commissioned by the Vlaamse Waterweg NV. Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>|
|\n", perl = TRUE)) +paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] + +# Split paragraph +paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) +paragraphs <- purrr::map(paragraphs, ~ if(grepl("^ the European", .x)) { + .x <- paste0("Data were exported from", .x) + } else {.x <- .x}) + +# Rearrange paragraphs: abstract + additional_info +eml$dataset$abstract$para <- c(eml$dataset$abstract$para, paragraphs) + +# Delete `additionalInfo` +eml$dataset$additionalInfo <- NULL +``` + +### Clean Resource contacts and Resource creators + +- Generic email "info@inbo.be" is removed. +- Organization "Vlaamse overheid; Beleidsdomein Omgeving; Instituut voor Natuur- en Bosonderzoek" is replaced by "Research Institute for Nature and Forest (INBO)". ```{r} update_contact <- function(contact) { @@ -59,7 +92,7 @@ eml$dataset$creator <- purrr::map(eml$dataset$creator, update_contact) eml$dataset$contact <- update_contact(eml$dataset$contact) ``` -## Set metadataProvider +### Set Metadata provider ```{r} eml$dataset$metadataProvider <- EML::set_responsibleParty( @@ -74,20 +107,17 @@ eml$dataset$metadataProvider <- EML::set_responsibleParty( ) ``` -## Set generalTaxonomicCoverage and associatedParty to NULL +## Taxonomic coverage + +Remove description. ```{r} eml$dataset$coverage$taxonomicCoverage$generalTaxonomicCoverage <- NULL -eml$dataset$associatedParty <- NULL ``` -## Set maintenanceUpdateFrequency to "Not planned" +## Keywords -```{r} -eml$dataset$maintenance$maintenanceUpdateFrequency <- "Not planned" -``` - -## clean keywords +Clean keywords. ```{r} eml$dataset$keywordSet <- purrr::map(eml$dataset$keywordSet, ~ { @@ -98,40 +128,23 @@ eml$dataset$keywordSet <- purrr::map(eml$dataset$keywordSet, ~ { }) ``` -# Clean Reserouce Citation Identifier +## Associated Parties + +Remove associated parties. ```{r} -identifier <- eml$additionalMetadata$metadata$gbif$citation$identifier -eml$additionalMetadata$metadata$gbif$citation$identifier <- sub("dx.", "", identifier) +eml$dataset$associatedParty <- NULL ``` +## Citations -## Update abstract - -The abstract on IMIS is limited to 1000 characters. Extra paragraphs are living in `additionalInfo`. -We move the `additionalInfo` to the abstract and clean it up. +Clean Resource citation identifier ```{r} -additional_info <- eml$dataset$additionalInfo$para - -# Split `additional_info` in paragraphs -paragraphs <- unlist(strsplit(additional_info, "|
|\n", perl = TRUE)) -paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] - -# Split paragraph -paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) -paragraphs <- purrr::map(paragraphs, ~ if(grepl("^ the European", .x)) { - .x <- paste0("Data were exported from", .x) - } else {.x <- .x}) - -# Rearrange paragraphs: abstract + additional_info -eml$dataset$abstract$para <- c(eml$dataset$abstract$para, paragraphs) - -# Delete `additionalInfo` -eml$dataset$additionalInfo <- NULL +identifier <- eml$additionalMetadata$metadata$gbif$citation$identifier +eml$additionalMetadata$metadata$gbif$citation$identifier <- sub("dx.", "", identifier) ``` - # Write EML ```{r} From e4ea86e8d931b075ef0b699050bed858aef27260 Mon Sep 17 00:00:00 2001 From: Sanne Govaert <44606923+sannegovaert@users.noreply.github.com> Date: Fri, 13 Sep 2024 10:00:01 +0200 Subject: [PATCH 06/19] Clean Resource citation identifier --- data/2011_RIVIERPRIK/processed/eml.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/2011_RIVIERPRIK/processed/eml.xml b/data/2011_RIVIERPRIK/processed/eml.xml index 4f172b4..3094ede 100644 --- a/data/2011_RIVIERPRIK/processed/eml.xml +++ b/data/2011_RIVIERPRIK/processed/eml.xml @@ -6,7 +6,7 @@|
|\n", perl = TRUE)) -paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] - -# Split paragraph -paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) -paragraphs <- purrr::map(paragraphs, ~ if(grepl("^ the European", .x)) { - .x <- paste0("Data were exported from", .x) - } else {.x <- .x}) - -# Rearrange paragraphs: abstract + additional_info -eml$dataset$abstract$para <- c(eml$dataset$abstract$para, paragraphs) - -# Delete `additionalInfo` -eml$dataset$additionalInfo <- NULL -``` - ### Clean Resource contacts and Resource creators - Generic email "info@inbo.be" is removed. @@ -154,8 +130,44 @@ eml$dataset$associatedParty <- NULL Clean Resource citation identifier ```{r} -identifier <- eml$additionalMetadata$metadata$gbif$citation$identifier -eml$additionalMetadata$metadata$gbif$citation$identifier <- sub("dx.", "", identifier) +identifier_raw <- eml$additionalMetadata$metadata$gbif$citation$identifier +identifier <- sub("dx.", "", identifier_raw) +eml$additionalMetadata$metadata$gbif$citation$identifier <- identifier +``` +## Update Description in Metadata + +The abstract on IMIS is limited to 1000 characters. Extra paragraphs are living in `additionalInfo` (Additional metadata). +We move the `additionalInfo` to the abstract and clean it up. + +```{r} +# Get additional paragraphs +additional_info <- eml$dataset$additionalInfo$para + +# Split `additional_info` in paragraphs +paragraphs <- unlist(strsplit(additional_info, "|
|\n", perl = TRUE)) +paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] +paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) + +# Extract publication year +citation <- eml$additionalMetadata$metadata$gbif$citation$citation +pattern <- "\\(\\d{4}\\)" +year <- stringr::str_extract(citation, pattern) %>% + stringr::str_remove_all("\\(|\\)") + +# Extract first author +first_author <- stringr::str_split_1(citation, ",")[1] + +# Write new paragraph +new_paragraph <- paste0("Data have been standardized to Darwin Core using the etn package and are downsampled to the first detection per hour. The original data are managed in the European Tracking Network data platform (https://www.lifewatch.be/etn/) and are available in ", first_author, " et al. (", year, ", ", identifier, ").") + +# Update last paragraph +paragraphs[length(paragraphs)] <- new_paragraph + +# Rearrange paragraphs: abstract + additional_info +eml$dataset$abstract$para <- c(eml$dataset$abstract$para, paragraphs) + +# Delete `additionalInfo` +eml$dataset$additionalInfo <- NULL ``` # Write EML From f70b4924e59636b03c0b416c0b4b09a7a8ae1ce1 Mon Sep 17 00:00:00 2001 From: Sanne Govaert <44606923+sannegovaert@users.noreply.github.com> Date: Tue, 17 Sep 2024 15:25:30 +0200 Subject: [PATCH 11/19] update description --- src/prepare_eml_for_GBIF.Rmd | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/prepare_eml_for_GBIF.Rmd b/src/prepare_eml_for_GBIF.Rmd index 15c23da..02b74de 100644 --- a/src/prepare_eml_for_GBIF.Rmd +++ b/src/prepare_eml_for_GBIF.Rmd @@ -148,6 +148,12 @@ paragraphs <- unlist(strsplit(additional_info, "|
|\n", perl = TRUE)) paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) +# Wrap paragraphs with html with ") + } else {.x <- .x} +) + # Extract publication year citation <- eml$additionalMetadata$metadata$gbif$citation$citation pattern <- "\\(\\d{4}\\)" @@ -158,7 +164,7 @@ year <- stringr::str_extract(citation, pattern) %>% first_author <- stringr::str_split_1(citation, ",")[1] # Write new paragraph -new_paragraph <- paste0("Data have been standardized to Darwin Core using the etn package and are downsampled to the first detection per hour. The original data are managed in the European Tracking Network data platform (https://www.lifewatch.be/etn/) and are available in ", first_author, " et al. (", year, ", ", identifier, ").") +new_paragraph <- paste0("Data have been standardized to Darwin Core using the etn package and are downsampled to the first detection per hour. The original data are managed in the European Tracking Network data platform (https://www.lifewatch.be/etn/) and are available in ", first_author, " et al. (", year, ", ", identifier, ").") # Update last paragraph paragraphs[length(paragraphs)] <- new_paragraph From 0f762d545736b96342d9edaf8a357e6173e3735b Mon Sep 17 00:00:00 2001 From: Peter Desmet|
|\n", perl = TRUE)) paragraphs <- paragraphs[!paragraphs %in% c("", "", "]]>")] paragraphs <- unlist(strsplit(paragraphs, " Data were exported from", perl = TRUE)) -# Wrap paragraphs with html with paragraphs <- purrr::map(paragraphs, ~ if(grepl("<", .)){ .x <- paste0("") } else {.x <- .x} @@ -179,8 +177,6 @@ eml$dataset$additionalInfo <- NULL # Write EML ```{r} -eml_path <- here::here("data", project_id, "processed", "eml.xml") +eml_path <- here::here("data", "processed", project_id, "eml.xml") EML::write_eml(eml, eml_path) ``` - - From 98f0d969e5e937ca6b8d85f07e6190b53eeb092c Mon Sep 17 00:00:00 2001 From: Peter DesmetThe study was commissioned by the Vlaamse Waterweg NV. Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>This dataset was collected using infrastructure provided by VLIZ and INBO funded by the Research Foundation - Flanders (FWO) as part of the Belgian contribution to LifeWatch. The study was commissioned by the Agency of Nature and Forest (ANB). Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>This dataset was collected using infrastructure provided by VLIZ and INBO funded by the Research Foundation - Flanders (FWO) as part of the Belgian contribution to LifeWatch. The study was commissioned by the Vlaamse Waterweg NV. Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>This dataset was collected using infrastructure provided by VLIZ and INBO funded by the Research Foundation - Flanders (FWO) as part of the Belgian contribution to LifeWatch. The study was commissioned by the Vlaamse Waterweg NV. Data were exported from the European Tracking Network data portal (http://www.lifewatch.be/etn) developed by VLIZ using the ETN R package (https://inbo.github.io/etn/) developed by INBO. Field definitions can be found at https://inbo.github.io/etn/articles/etn_fields.html.
]]>