Merge branch 'f-cb-2'

- Implement continuous benchmarking (#793). Closes #793.
tidyverse · Jul 5, 2020 · f01d8da · f01d8da
2 parents 2a5d22c + 917df59
commit f01d8da
Show file tree

Hide file tree

Showing 6 changed files with 132 additions and 84 deletions.
diff --git a/.Rbuildignore b/.Rbuildignore
@@ -11,6 +11,8 @@
 ^revdep$
 ^README-.*\.png$
 ^appveyor\.yml$
+^bench$
+^\.github$
 ^\.deploy_key\.pub$
 ^\.deploy_key\.enc$
 ^API$

diff --git a/.gitattributes b/.gitattributes
@@ -1 +1,2 @@
 /NAMESPACE merge=union
+/.Rbuildignore merge=union
diff --git a/.github/workflows/continuous-benchmarks.yaml b/.github/workflows/continuous-benchmarks.yaml
@@ -0,0 +1,36 @@
+on: push
+
+name: Continuous Benchmarks
+
+jobs:
+  build:
+    runs-on: macOS-latest
+    steps:
+      - name: Checkout repo
+        uses: actions/checkout@master
+
+      - name: Setup R
+        uses: r-lib/actions/setup-r@master
+
+      - name: Install dependencies
+        run: |
+          install.packages("remotes")
+          remotes::install_deps(dependencies = TRUE)
+          remotes::install_github("r-lib/bench")
+          install.packages("here")
+        shell: Rscript {0}
+
+      - name: Install package
+        run: R CMD INSTALL .
+
+      - name: Fetch existing benchmarks
+        run: Rscript -e 'bench::cb_fetch()' || true
+
+      - name: Run benchmarks
+        run: Rscript -e 'bench::cb_run()'
+
+      - name: Show benchmarks
+        run: git notes --ref benchmarks show
+
+      - name: Push benchmarks
+        run: Rscript -e "bench::cb_push()"
diff --git a/bench/bench-subsetting.R b/bench/bench-subsetting.R
@@ -1,89 +1,8 @@
 library(tibble)
-library(dplyr)
 library(bench)
+library(here)
 
-bm <- function(df) {
-  idx <- c(rep(TRUE, 5), rep(FALSE, 5))
-  bench::mark(
-    df[[1]], df[[1]] <- 1,
-    df[["a"]], df[["a"]] <- 1,
-    df[["aa"]], df[["aa"]] <- 1,
-    df[["aaa"]], df[["aaa"]] <- 1,
-    df[["b"]],
-
-    df$a, df$a <- 1,
-    df$aa, df$aa <- 1,
-    df$aaa, df$aaa <- 1,
-
-    df["a"], df["a"] <- 1,
-    df["aa"], df["aa"] <- 1,
-    df[c("aa", "aaa")], df[c("aa", "aaa")] <- 1,
-    df[1], df[1] <- 1,
-    df[2:3], df[2:3] <- 1,
-    df[TRUE], df[TRUE] <- 1,
-    df[c(TRUE, FALSE, TRUE)], df[c(TRUE, FALSE, TRUE)] <- 1,
-
-    df[, "a"], df[, "a"] <- 1,
-    df[, "aa"], df[, "aa"] <- 1,
-    df[, c("aa", "aaa")], df[, c("aa", "aaa")] <- 1,
-    df[, 1], df[, 1] <- 1,
-    df[, 2:3], df[, 2:3] <- 1,
-    df[, TRUE], df[, TRUE] <- 1,
-    df[, c(TRUE, FALSE, TRUE)], df[, c(TRUE, FALSE, TRUE)] <- 1,
-
-    df[1, ], df[1, ] <- 1,
-    df[3:7, ], df[3:7, ] <- 1,
-    df[TRUE, ], df[TRUE, ] <- 1,
-    df[idx, ], df[idx, ] <- 1,
-
-    df[1, "a"], df[1, "a"] <- 1,
-    df[1, "aa"], df[1, "aa"] <- 1,
-    df[1, c("aa", "aaa")], df[1, c("aa", "aaa")] <- 1,
-    df[1, 1], df[1, 1] <- 1,
-    df[1, 2:3], df[1, 2:3] <- 1,
-    df[1, TRUE], df[1, TRUE] <- 1,
-    df[1, c(TRUE, FALSE, TRUE)], df[1, c(TRUE, FALSE, TRUE)] <- 1,
-
-    df[3:7, "a"], df[3:7, "a"] <- 1,
-    df[3:7, "aa"], df[3:7, "aa"] <- 1,
-    df[3:7, c("aa", "aaa")], df[3:7, c("aa", "aaa")] <- 1,
-    df[3:7, 1], df[3:7, 1] <- 1,
-    df[3:7, 2:3], df[3:7, 2:3] <- 1,
-    df[3:7, TRUE], df[3:7, TRUE] <- 1,
-    df[3:7, c(TRUE, FALSE, TRUE)], df[3:7, c(TRUE, FALSE, TRUE)] <- 1,
-
-    df[TRUE, "a"], df[TRUE, "a"] <- 1,
-    df[TRUE, "aa"], df[TRUE, "aa"] <- 1,
-    df[TRUE, c("aa", "aaa")], df[TRUE, c("aa", "aaa")] <- 1,
-    df[TRUE, 1], df[TRUE, 1] <- 1,
-    df[TRUE, 2:3], df[TRUE, 2:3] <- 1,
-    df[TRUE, TRUE], df[TRUE, TRUE] <- 1,
-    df[TRUE, c(TRUE, FALSE, TRUE)], df[TRUE, c(TRUE, FALSE, TRUE)] <- 1,
-
-    df[idx, "a"], df[idx, "a"] <- 1,
-    df[idx, "aa"], df[idx, "aa"] <- 1,
-    df[idx, c("aa", "aaa")], df[idx, c("aa", "aaa")] <- 1,
-    df[idx, 1], df[idx, 1] <- 1,
-    df[idx, 2:3], df[idx, 2:3] <- 1,
-    df[idx, TRUE], df[idx, TRUE] <- 1,
-    df[idx, c(TRUE, FALSE, TRUE)], df[idx, c(TRUE, FALSE, TRUE)] <- 1,
-
-    check = FALSE,
-    iterations = 2000
-  )
-}
+source(here("bench/fun/subsetting.R"))
 
 df <- tibble(a = 1:10, aa = 1:10, aaa = 1:10)
-b_tibble <- bm(df)
-
-df <- data.frame(a = 1:10, aa = 1:10, aaa = 1:10)
-b_df <- bm(df)
-
-b_df %>%
-  select(expression, median) %>%
-  left_join(b_tibble %>% select(expression, median), by = "expression") %>%
-  mutate(ratio = as.numeric(median.y / median.x)) %>%
-  arrange(-ratio) %>%
-  view()
-
-b_tibble %>% arrange(desc(median))
+bm(df)
diff --git a/bench/code/compare-df.R b/bench/code/compare-df.R
@@ -0,0 +1,20 @@
+library(tibble)
+library(dplyr)
+library(bench)
+
+source("bench/fun/subsetting.R")
+
+df <- tibble(a = 1:10, aa = 1:10, aaa = 1:10)
+b_tibble <- bm(df)
+
+df <- data.frame(a = 1:10, aa = 1:10, aaa = 1:10)
+b_df <- bm(df)
+
+b_df %>%
+  select(expression, median) %>%
+  left_join(b_tibble %>% select(expression, median), by = "expression") %>%
+  mutate(ratio = as.numeric(median.y / median.x)) %>%
+  arrange(-ratio) %>%
+  view()
+
+b_tibble %>% arrange(desc(median))
diff --git a/bench/fun/subsetting.R b/bench/fun/subsetting.R
@@ -0,0 +1,70 @@
+bm <- function(df) {
+  idx <- c(rep(TRUE, 5), rep(FALSE, 5))
+  bench::mark(
+    df[[1]], df[[1]] <- 1,
+    df[["a"]], df[["a"]] <- 1,
+    df[["aa"]], df[["aa"]] <- 1,
+    df[["aaa"]], df[["aaa"]] <- 1,
+    df[["b"]],
+
+    df$a, df$a <- 1,
+    df$aa, df$aa <- 1,
+    df$aaa, df$aaa <- 1,
+
+    df["a"], df["a"] <- 1,
+    df["aa"], df["aa"] <- 1,
+    df[c("aa", "aaa")], df[c("aa", "aaa")] <- 1,
+    df[1], df[1] <- 1,
+    df[2:3], df[2:3] <- 1,
+    df[TRUE], df[TRUE] <- 1,
+    df[c(TRUE, FALSE, TRUE)], df[c(TRUE, FALSE, TRUE)] <- 1,
+
+    df[, "a"], df[, "a"] <- 1,
+    df[, "aa"], df[, "aa"] <- 1,
+    df[, c("aa", "aaa")], df[, c("aa", "aaa")] <- 1,
+    df[, 1], df[, 1] <- 1,
+    df[, 2:3], df[, 2:3] <- 1,
+    df[, TRUE], df[, TRUE] <- 1,
+    df[, c(TRUE, FALSE, TRUE)], df[, c(TRUE, FALSE, TRUE)] <- 1,
+
+    df[1, ], df[1, ] <- 1,
+    df[3:7, ], df[3:7, ] <- 1,
+    df[TRUE, ], df[TRUE, ] <- 1,
+    df[idx, ], df[idx, ] <- 1,
+
+    df[1, "a"], df[1, "a"] <- 1,
+    df[1, "aa"], df[1, "aa"] <- 1,
+    df[1, c("aa", "aaa")], df[1, c("aa", "aaa")] <- 1,
+    df[1, 1], df[1, 1] <- 1,
+    df[1, 2:3], df[1, 2:3] <- 1,
+    df[1, TRUE], df[1, TRUE] <- 1,
+    df[1, c(TRUE, FALSE, TRUE)], df[1, c(TRUE, FALSE, TRUE)] <- 1,
+
+    df[3:7, "a"], df[3:7, "a"] <- 1,
+    df[3:7, "aa"], df[3:7, "aa"] <- 1,
+    df[3:7, c("aa", "aaa")], df[3:7, c("aa", "aaa")] <- 1,
+    df[3:7, 1], df[3:7, 1] <- 1,
+    df[3:7, 2:3], df[3:7, 2:3] <- 1,
+    df[3:7, TRUE], df[3:7, TRUE] <- 1,
+    df[3:7, c(TRUE, FALSE, TRUE)], df[3:7, c(TRUE, FALSE, TRUE)] <- 1,
+
+    df[TRUE, "a"], df[TRUE, "a"] <- 1,
+    df[TRUE, "aa"], df[TRUE, "aa"] <- 1,
+    df[TRUE, c("aa", "aaa")], df[TRUE, c("aa", "aaa")] <- 1,
+    df[TRUE, 1], df[TRUE, 1] <- 1,
+    df[TRUE, 2:3], df[TRUE, 2:3] <- 1,
+    df[TRUE, TRUE], df[TRUE, TRUE] <- 1,
+    df[TRUE, c(TRUE, FALSE, TRUE)], df[TRUE, c(TRUE, FALSE, TRUE)] <- 1,
+
+    df[idx, "a"], df[idx, "a"] <- 1,
+    df[idx, "aa"], df[idx, "aa"] <- 1,
+    df[idx, c("aa", "aaa")], df[idx, c("aa", "aaa")] <- 1,
+    df[idx, 1], df[idx, 1] <- 1,
+    df[idx, 2:3], df[idx, 2:3] <- 1,
+    df[idx, TRUE], df[idx, TRUE] <- 1,
+    df[idx, c(TRUE, FALSE, TRUE)], df[idx, c(TRUE, FALSE, TRUE)] <- 1,
+
+    check = FALSE,
+    iterations = 2000
+  )
+}
Original file line number	Diff line number	Diff line change
		@@ -1 +1,2 @@
		/NAMESPACE merge=union
		/.Rbuildignore merge=union