sending to CRAN with fix suggested by Ivan Krylov

rpkg/.Rbuildignore

@@ -13,7 +13,6 @@
 ^vignettes/CP2017CHL$
 ^\.covrignore$
 ^cran-comments\.md$
-^inst/extdata/galapagos$
 ^_pkgdown\.yml$
 ^microdata$
 ^redatam$

rpkg/.gitignore

@@ -1,6 +1,5 @@
 .vscode
 dev
-inst/extdata/galapagos
 src/*.o
 src/*.so
 src/redatamlib/*.o

rpkg/DESCRIPTION

@@ -13,15 +13,19 @@ Authors@R: c(
         given = "Lital",
         family = "Barkai",
         role = "aut"),
+    person(
+        given = "Ivan",
+        family = "Krylov",
+        role = "ctb"),
     person(
         given = "Arseny",
         family = "Kapoulkine",
         role = "ctb",
         comment = "'pugixml' C++ library"),
     person(
-        family = "Republic of Ecuador",
+        family = "Oriental Republic of Uruguay",
         role = "dtc",
-        comment = "Galapagos census data")
+        comment = "Aggregated census data from 2011")
     )
 Imports:
     data.table,

rpkg/NEWS.md

@@ -1,19 +1,10 @@
-# redatam 2.0.4
+# redatam 2.0.1
 
 * Refactored C++ to R list casting to avoid growing a list in a loop. It now
   creates a list with a lenght equal to the number of entities and variables
   with descriptions and then fills it with the data. This is 3 to 5 times
   faster.
-
-# redatam 2.0.3
-
-* Fixes memory management issues suggested by Ivan Krylov regarding the C++ to R
-  list casting.
-* Uses 2 threads during R CMD check
-
-# redatam 2.0.1
-
-* Fixes memory leaks warned by CRAN on clang-ASAN and gcc-UBSAN.
+* Fixes GCC-SAN errors with a fix proposed by Ivan Krylov. Thanks!
 
 # redatam 2.0.0
 

rpkg/R/read_redatam.R

@@ -5,11 +5,13 @@
 #' @return A list of tibbles, each one representing a table with the census
 #'  entities (or levels) and their attributes (or variables).
 #' @examples
-#' # Read a dictionary file (Galapagos 2015)
-#' zip <- system.file("extdata", "galapagos.zip", package = "redatam")
-#' dout <- paste(tempdir(), "galapagos", sep = "/")
+#' # Read a dictionary file (Uruguay 2011, aggregated)
+#' zip <- system.file("extdata", "uru2011mini.zip", package = "redatam")
+#' dout <- paste(tempdir(), "uru2011mini", sep = "/")
 #' unzip(zip, exdir = dout)
-#' read_redatam(paste(dout, "cg15.dic", sep = "/"))
+#' read_redatam(paste(dout, "uru2011mini.dic", sep = "/"))
+#' # or equivalently
+#' read_redatam(paste(dout, "uru2011mini.dicx", sep = "/"))
 #' @export
 read_redatam <- function(dictionary) {
   dictionary <- normalizePath(dictionary)

rpkg/src/redatamlib/readers/ByteArrayReader.cpp

@@ -1,6 +1,6 @@
 #include <algorithm>  // std::search
 #include <fstream>    // std::ifstream
-#include <iterator>  //  std::ostream_iterator, std::istreambuf_iterator
+#include <iterator>   // std::ostream_iterator, std::istreambuf_iterator
 #include <sstream>    // std::ostringstream
 
 #include "ByteArrayReader.hpp"
@@ -175,23 +175,27 @@ unsigned char ByteArrayReader::ReadByte() {
 }
 
 uint16_t ByteArrayReader::ReadInt16LE() {
-  return static_cast<uint16_t>(ReadByte()) |
-         (static_cast<uint16_t>(ReadByte()) << 8);
+  uint16_t a = static_cast<uint16_t>(ReadByte());
+  uint16_t b = static_cast<uint16_t>(ReadByte()) << 8;
+  return a | b;
 }
 
 uint32_t ByteArrayReader::ReadInt32LE() {
-  return static_cast<uint32_t>(ReadInt16LE()) |
-         static_cast<uint32_t>(ReadInt16LE()) << 16;
+  uint32_t a = static_cast<uint32_t>(ReadInt16LE());
+  uint32_t b = static_cast<uint32_t>(ReadInt16LE()) << 16;
+  return a | b;
 }
 
 uint16_t ByteArrayReader::ReadInt16BE() {
-  return (static_cast<uint16_t>(ReadByte()) << 8) |
-         static_cast<uint16_t>(ReadByte());
+  uint16_t a = static_cast<uint16_t>(ReadByte()) << 8;
+  uint16_t b = static_cast<uint16_t>(ReadByte());
+  return a | b;
 }
 
 uint32_t ByteArrayReader::ReadInt32BE() {
-  return (static_cast<uint32_t>(ReadInt16BE()) << 16) |
-         static_cast<uint32_t>(ReadInt16BE());
+  uint32_t a = static_cast<uint32_t>(ReadInt16BE()) << 16;
+  uint32_t b = static_cast<uint32_t>(ReadInt16BE());
+  return a | b;
 }
 
 }  // namespace RedatamLib

rpkg/tests/testthat/test-uruguay.R

@@ -0,0 +1,59 @@
+# the Galapagos census was downloaded from
+# https://redatam.org/cdr/descargas/censos/poblacion/CP2011URY.zip
+# I agregated the data to test the code
+
+# read DICX
+# when converting the original DIC to DICX in REDATAM 7:
+# 1. it doesn't include all the labels
+# 2. it adds MISSING and NOTAPPLICABLE to the labels
+# 3. it creates problems with lower and upper case (e.g., CG150001.ptr and
+#    cg150001.ptr)
+
+test_that("reading works", {
+  zip <- system.file("extdata", "uru2011mini.zip", package = "redatam")
+
+  dout <- paste(tempdir(), "uru2011mini", sep = "/")
+
+  if (file.exists(dout)) {
+    unlink(dout, recursive = T)
+  }
+
+  # unzip the file
+  if (!file.exists(dout)) {
+    unzip(zip, exdir = dout)
+  }
+
+  # find the dictionary
+  dic <- list.files(dout, pattern = "\\.dic$", full.names = TRUE,
+    recursive = TRUE)
+
+  dicx <- list.files(dout, pattern = "\\.dicx$", full.names = TRUE,
+    recursive = TRUE)
+
+  # read DIC
+
+  res <- read_redatam(dic)
+
+  expect_type(res, "list")
+  expect_equal(length(res), 3L)
+
+  d <- res$sexo
+  expect_true(is.data.frame(d))
+  expect_true(is.factor(d$sexo))
+  expect_equal(dim(d), c(38L,4L))
+
+  daux <- res$sexo_labels_cuenta
+  expect_true(is.factor(daux$cuenta_description))
+  expect_equal(dim(daux), c(2L,2L))
+
+  # read DICX
+
+  res2 <- read_redatam(dicx)
+
+  expect_type(res2, "list")
+  expect_equal(length(res2), 2L)
+
+  d2 <- res2$sexo
+  expect_true(is.data.frame(d2))
+  expect_equal(dim(d2), c(38L,4L))
+})