From 273129fd6120b3c9c5613022058a73e9cf79fa8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 05:05:15 +0200 Subject: [PATCH 1/7] Import --- .Rbuildignore | 1 + bench/bench-subsetting.R | 89 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+) create mode 100644 bench/bench-subsetting.R diff --git a/.Rbuildignore b/.Rbuildignore index ba6e4aa30..bcc2665aa 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -8,3 +8,4 @@ ^revdep$ ^README-.*\.png$ ^appveyor\.yml$ +^bench$ diff --git a/bench/bench-subsetting.R b/bench/bench-subsetting.R new file mode 100644 index 000000000..a90d13422 --- /dev/null +++ b/bench/bench-subsetting.R @@ -0,0 +1,89 @@ +library(tibble) +library(dplyr) +library(bench) + +bm <- function(df) { + idx <- c(rep(TRUE, 5), rep(FALSE, 5)) + bench::mark( + df[[1]], df[[1]] <- 1, + df[["a"]], df[["a"]] <- 1, + df[["aa"]], df[["aa"]] <- 1, + df[["aaa"]], df[["aaa"]] <- 1, + df[["b"]], + + df$a, df$a <- 1, + df$aa, df$aa <- 1, + df$aaa, df$aaa <- 1, + + df["a"], df["a"] <- 1, + df["aa"], df["aa"] <- 1, + df[c("aa", "aaa")], df[c("aa", "aaa")] <- 1, + df[1], df[1] <- 1, + df[2:3], df[2:3] <- 1, + df[TRUE], df[TRUE] <- 1, + df[c(TRUE, FALSE, TRUE)], df[c(TRUE, FALSE, TRUE)] <- 1, + + df[, "a"], df[, "a"] <- 1, + df[, "aa"], df[, "aa"] <- 1, + df[, c("aa", "aaa")], df[, c("aa", "aaa")] <- 1, + df[, 1], df[, 1] <- 1, + df[, 2:3], df[, 2:3] <- 1, + df[, TRUE], df[, TRUE] <- 1, + df[, c(TRUE, FALSE, TRUE)], df[, c(TRUE, FALSE, TRUE)] <- 1, + + df[1, ], df[1, ] <- 1, + df[3:7, ], df[3:7, ] <- 1, + df[TRUE, ], df[TRUE, ] <- 1, + df[idx, ], df[idx, ] <- 1, + + df[1, "a"], df[1, "a"] <- 1, + df[1, "aa"], df[1, "aa"] <- 1, + df[1, c("aa", "aaa")], df[1, c("aa", "aaa")] <- 1, + df[1, 1], df[1, 1] <- 1, + df[1, 2:3], df[1, 2:3] <- 1, + df[1, TRUE], df[1, TRUE] <- 1, + df[1, c(TRUE, FALSE, TRUE)], df[1, c(TRUE, FALSE, TRUE)] <- 1, + + df[3:7, "a"], df[3:7, "a"] <- 1, + df[3:7, "aa"], df[3:7, "aa"] <- 1, + df[3:7, c("aa", "aaa")], df[3:7, c("aa", "aaa")] <- 1, + df[3:7, 1], df[3:7, 1] <- 1, + df[3:7, 2:3], df[3:7, 2:3] <- 1, + df[3:7, TRUE], df[3:7, TRUE] <- 1, + df[3:7, c(TRUE, FALSE, TRUE)], df[3:7, c(TRUE, FALSE, TRUE)] <- 1, + + df[TRUE, "a"], df[TRUE, "a"] <- 1, + df[TRUE, "aa"], df[TRUE, "aa"] <- 1, + df[TRUE, c("aa", "aaa")], df[TRUE, c("aa", "aaa")] <- 1, + df[TRUE, 1], df[TRUE, 1] <- 1, + df[TRUE, 2:3], df[TRUE, 2:3] <- 1, + df[TRUE, TRUE], df[TRUE, TRUE] <- 1, + df[TRUE, c(TRUE, FALSE, TRUE)], df[TRUE, c(TRUE, FALSE, TRUE)] <- 1, + + df[idx, "a"], df[idx, "a"] <- 1, + df[idx, "aa"], df[idx, "aa"] <- 1, + df[idx, c("aa", "aaa")], df[idx, c("aa", "aaa")] <- 1, + df[idx, 1], df[idx, 1] <- 1, + df[idx, 2:3], df[idx, 2:3] <- 1, + df[idx, TRUE], df[idx, TRUE] <- 1, + df[idx, c(TRUE, FALSE, TRUE)], df[idx, c(TRUE, FALSE, TRUE)] <- 1, + + check = FALSE, + iterations = 2000 + ) +} + +df <- tibble(a = 1:10, aa = 1:10, aaa = 1:10) +b_tibble <- bm(df) + +df <- data.frame(a = 1:10, aa = 1:10, aaa = 1:10) +b_df <- bm(df) + +b_df %>% + select(expression, median) %>% + left_join(b_tibble %>% select(expression, median), by = "expression") %>% + mutate(ratio = as.numeric(median.y / median.x)) %>% + arrange(-ratio) %>% + view() + +b_tibble %>% arrange(desc(median)) From 42891e0258f999efb0e2dcba54e2d7114fff8ae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 05:10:26 +0200 Subject: [PATCH 2/7] Separate measurement and comparison code --- bench/bench-subsetting.R | 86 +--------------------------------------- bench/code/compare-df.R | 20 ++++++++++ bench/fun/subsetting.R | 70 ++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 84 deletions(-) create mode 100644 bench/code/compare-df.R create mode 100644 bench/fun/subsetting.R diff --git a/bench/bench-subsetting.R b/bench/bench-subsetting.R index a90d13422..898c04e4e 100644 --- a/bench/bench-subsetting.R +++ b/bench/bench-subsetting.R @@ -1,89 +1,7 @@ library(tibble) -library(dplyr) library(bench) -bm <- function(df) { - idx <- c(rep(TRUE, 5), rep(FALSE, 5)) - bench::mark( - df[[1]], df[[1]] <- 1, - df[["a"]], df[["a"]] <- 1, - df[["aa"]], df[["aa"]] <- 1, - df[["aaa"]], df[["aaa"]] <- 1, - df[["b"]], - - df$a, df$a <- 1, - df$aa, df$aa <- 1, - df$aaa, df$aaa <- 1, - - df["a"], df["a"] <- 1, - df["aa"], df["aa"] <- 1, - df[c("aa", "aaa")], df[c("aa", "aaa")] <- 1, - df[1], df[1] <- 1, - df[2:3], df[2:3] <- 1, - df[TRUE], df[TRUE] <- 1, - df[c(TRUE, FALSE, TRUE)], df[c(TRUE, FALSE, TRUE)] <- 1, - - df[, "a"], df[, "a"] <- 1, - df[, "aa"], df[, "aa"] <- 1, - df[, c("aa", "aaa")], df[, c("aa", "aaa")] <- 1, - df[, 1], df[, 1] <- 1, - df[, 2:3], df[, 2:3] <- 1, - df[, TRUE], df[, TRUE] <- 1, - df[, c(TRUE, FALSE, TRUE)], df[, c(TRUE, FALSE, TRUE)] <- 1, - - df[1, ], df[1, ] <- 1, - df[3:7, ], df[3:7, ] <- 1, - df[TRUE, ], df[TRUE, ] <- 1, - df[idx, ], df[idx, ] <- 1, - - df[1, "a"], df[1, "a"] <- 1, - df[1, "aa"], df[1, "aa"] <- 1, - df[1, c("aa", "aaa")], df[1, c("aa", "aaa")] <- 1, - df[1, 1], df[1, 1] <- 1, - df[1, 2:3], df[1, 2:3] <- 1, - df[1, TRUE], df[1, TRUE] <- 1, - df[1, c(TRUE, FALSE, TRUE)], df[1, c(TRUE, FALSE, TRUE)] <- 1, - - df[3:7, "a"], df[3:7, "a"] <- 1, - df[3:7, "aa"], df[3:7, "aa"] <- 1, - df[3:7, c("aa", "aaa")], df[3:7, c("aa", "aaa")] <- 1, - df[3:7, 1], df[3:7, 1] <- 1, - df[3:7, 2:3], df[3:7, 2:3] <- 1, - df[3:7, TRUE], df[3:7, TRUE] <- 1, - df[3:7, c(TRUE, FALSE, TRUE)], df[3:7, c(TRUE, FALSE, TRUE)] <- 1, - - df[TRUE, "a"], df[TRUE, "a"] <- 1, - df[TRUE, "aa"], df[TRUE, "aa"] <- 1, - df[TRUE, c("aa", "aaa")], df[TRUE, c("aa", "aaa")] <- 1, - df[TRUE, 1], df[TRUE, 1] <- 1, - df[TRUE, 2:3], df[TRUE, 2:3] <- 1, - df[TRUE, TRUE], df[TRUE, TRUE] <- 1, - df[TRUE, c(TRUE, FALSE, TRUE)], df[TRUE, c(TRUE, FALSE, TRUE)] <- 1, - - df[idx, "a"], df[idx, "a"] <- 1, - df[idx, "aa"], df[idx, "aa"] <- 1, - df[idx, c("aa", "aaa")], df[idx, c("aa", "aaa")] <- 1, - df[idx, 1], df[idx, 1] <- 1, - df[idx, 2:3], df[idx, 2:3] <- 1, - df[idx, TRUE], df[idx, TRUE] <- 1, - df[idx, c(TRUE, FALSE, TRUE)], df[idx, c(TRUE, FALSE, TRUE)] <- 1, - - check = FALSE, - iterations = 2000 - ) -} +source("bench/fun/subsetting.R") df <- tibble(a = 1:10, aa = 1:10, aaa = 1:10) -b_tibble <- bm(df) - -df <- data.frame(a = 1:10, aa = 1:10, aaa = 1:10) -b_df <- bm(df) - -b_df %>% - select(expression, median) %>% - left_join(b_tibble %>% select(expression, median), by = "expression") %>% - mutate(ratio = as.numeric(median.y / median.x)) %>% - arrange(-ratio) %>% - view() - -b_tibble %>% arrange(desc(median)) +bm(df) diff --git a/bench/code/compare-df.R b/bench/code/compare-df.R new file mode 100644 index 000000000..763f441ff --- /dev/null +++ b/bench/code/compare-df.R @@ -0,0 +1,20 @@ +library(tibble) +library(dplyr) +library(bench) + +source("bench/fun/subsetting.R") + +df <- tibble(a = 1:10, aa = 1:10, aaa = 1:10) +b_tibble <- bm(df) + +df <- data.frame(a = 1:10, aa = 1:10, aaa = 1:10) +b_df <- bm(df) + +b_df %>% + select(expression, median) %>% + left_join(b_tibble %>% select(expression, median), by = "expression") %>% + mutate(ratio = as.numeric(median.y / median.x)) %>% + arrange(-ratio) %>% + view() + +b_tibble %>% arrange(desc(median)) diff --git a/bench/fun/subsetting.R b/bench/fun/subsetting.R new file mode 100644 index 000000000..65ddb78e8 --- /dev/null +++ b/bench/fun/subsetting.R @@ -0,0 +1,70 @@ +bm <- function(df) { + idx <- c(rep(TRUE, 5), rep(FALSE, 5)) + bench::mark( + df[[1]], df[[1]] <- 1, + df[["a"]], df[["a"]] <- 1, + df[["aa"]], df[["aa"]] <- 1, + df[["aaa"]], df[["aaa"]] <- 1, + df[["b"]], + + df$a, df$a <- 1, + df$aa, df$aa <- 1, + df$aaa, df$aaa <- 1, + + df["a"], df["a"] <- 1, + df["aa"], df["aa"] <- 1, + df[c("aa", "aaa")], df[c("aa", "aaa")] <- 1, + df[1], df[1] <- 1, + df[2:3], df[2:3] <- 1, + df[TRUE], df[TRUE] <- 1, + df[c(TRUE, FALSE, TRUE)], df[c(TRUE, FALSE, TRUE)] <- 1, + + df[, "a"], df[, "a"] <- 1, + df[, "aa"], df[, "aa"] <- 1, + df[, c("aa", "aaa")], df[, c("aa", "aaa")] <- 1, + df[, 1], df[, 1] <- 1, + df[, 2:3], df[, 2:3] <- 1, + df[, TRUE], df[, TRUE] <- 1, + df[, c(TRUE, FALSE, TRUE)], df[, c(TRUE, FALSE, TRUE)] <- 1, + + df[1, ], df[1, ] <- 1, + df[3:7, ], df[3:7, ] <- 1, + df[TRUE, ], df[TRUE, ] <- 1, + df[idx, ], df[idx, ] <- 1, + + df[1, "a"], df[1, "a"] <- 1, + df[1, "aa"], df[1, "aa"] <- 1, + df[1, c("aa", "aaa")], df[1, c("aa", "aaa")] <- 1, + df[1, 1], df[1, 1] <- 1, + df[1, 2:3], df[1, 2:3] <- 1, + df[1, TRUE], df[1, TRUE] <- 1, + df[1, c(TRUE, FALSE, TRUE)], df[1, c(TRUE, FALSE, TRUE)] <- 1, + + df[3:7, "a"], df[3:7, "a"] <- 1, + df[3:7, "aa"], df[3:7, "aa"] <- 1, + df[3:7, c("aa", "aaa")], df[3:7, c("aa", "aaa")] <- 1, + df[3:7, 1], df[3:7, 1] <- 1, + df[3:7, 2:3], df[3:7, 2:3] <- 1, + df[3:7, TRUE], df[3:7, TRUE] <- 1, + df[3:7, c(TRUE, FALSE, TRUE)], df[3:7, c(TRUE, FALSE, TRUE)] <- 1, + + df[TRUE, "a"], df[TRUE, "a"] <- 1, + df[TRUE, "aa"], df[TRUE, "aa"] <- 1, + df[TRUE, c("aa", "aaa")], df[TRUE, c("aa", "aaa")] <- 1, + df[TRUE, 1], df[TRUE, 1] <- 1, + df[TRUE, 2:3], df[TRUE, 2:3] <- 1, + df[TRUE, TRUE], df[TRUE, TRUE] <- 1, + df[TRUE, c(TRUE, FALSE, TRUE)], df[TRUE, c(TRUE, FALSE, TRUE)] <- 1, + + df[idx, "a"], df[idx, "a"] <- 1, + df[idx, "aa"], df[idx, "aa"] <- 1, + df[idx, c("aa", "aaa")], df[idx, c("aa", "aaa")] <- 1, + df[idx, 1], df[idx, 1] <- 1, + df[idx, 2:3], df[idx, 2:3] <- 1, + df[idx, TRUE], df[idx, TRUE] <- 1, + df[idx, c(TRUE, FALSE, TRUE)], df[idx, c(TRUE, FALSE, TRUE)] <- 1, + + check = FALSE, + iterations = 2000 + ) +} From f1e7512ec18d5b62641cbe1b1d759ec53bf681f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 05:13:18 +0200 Subject: [PATCH 3/7] here --- bench/bench-subsetting.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bench/bench-subsetting.R b/bench/bench-subsetting.R index 898c04e4e..22684e39a 100644 --- a/bench/bench-subsetting.R +++ b/bench/bench-subsetting.R @@ -1,7 +1,8 @@ library(tibble) library(bench) +library(here) -source("bench/fun/subsetting.R") +source(here("bench/fun/subsetting.R")) df <- tibble(a = 1:10, aa = 1:10, aaa = 1:10) bm(df) From 6865cb4415609e1f5d3229583ab0a5e5d8327ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 05:32:06 +0200 Subject: [PATCH 4/7] Add workflow from dplyr --- .Rbuildignore | 1 + .github/workflows/continuous-benchmarks.yaml | 35 ++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 .github/workflows/continuous-benchmarks.yaml diff --git a/.Rbuildignore b/.Rbuildignore index bcc2665aa..2be6569a9 100644 --- a/.Rbuildignore +++ b/.Rbuildignore @@ -9,3 +9,4 @@ ^README-.*\.png$ ^appveyor\.yml$ ^bench$ +^\.github$ diff --git a/.github/workflows/continuous-benchmarks.yaml b/.github/workflows/continuous-benchmarks.yaml new file mode 100644 index 000000000..ba87b19b9 --- /dev/null +++ b/.github/workflows/continuous-benchmarks.yaml @@ -0,0 +1,35 @@ +on: push + +name: Continuous Benchmarks + +jobs: + build: + runs-on: macOS-latest + steps: + - name: Checkout repo + uses: actions/checkout@master + + - name: Setup R + uses: r-lib/actions/setup-r@master + + - name: Install dependencies + run: | + install.packages("remotes") + remotes::install_deps(dependencies = TRUE) + remotes::install_github("r-lib/bench") + shell: Rscript {0} + + - name: Install package + run: R CMD INSTALL . + + - name: Fetch existing benchmarks + run: Rscript -e 'bench::cb_fetch()' + + - name: Run benchmarks + run: Rscript -e 'bench::cb_run()' + + - name: Show benchmarks + run: git notes --ref benchmarks show + + - name: Push benchmarks + run: Rscript -e "bench::cb_push()" From d110856aeb5c5862789f709177aa276eee8f7ee9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 05:34:31 +0200 Subject: [PATCH 5/7] Install here --- .github/workflows/continuous-benchmarks.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/continuous-benchmarks.yaml b/.github/workflows/continuous-benchmarks.yaml index ba87b19b9..a0977ac99 100644 --- a/.github/workflows/continuous-benchmarks.yaml +++ b/.github/workflows/continuous-benchmarks.yaml @@ -17,6 +17,7 @@ jobs: install.packages("remotes") remotes::install_deps(dependencies = TRUE) remotes::install_github("r-lib/bench") + install.packages("here") shell: Rscript {0} - name: Install package From ac6d38562e0dcef89c224b20e6074607207d82cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 16:03:54 +0200 Subject: [PATCH 6/7] Allow failure when fetching --- .github/workflows/continuous-benchmarks.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/continuous-benchmarks.yaml b/.github/workflows/continuous-benchmarks.yaml index a0977ac99..0fbb3cb43 100644 --- a/.github/workflows/continuous-benchmarks.yaml +++ b/.github/workflows/continuous-benchmarks.yaml @@ -24,7 +24,7 @@ jobs: run: R CMD INSTALL . - name: Fetch existing benchmarks - run: Rscript -e 'bench::cb_fetch()' + run: Rscript -e 'bench::cb_fetch()' || true - name: Run benchmarks run: Rscript -e 'bench::cb_run()' From 6b2341119ee87785d728e5cc3b056a6a2ddb250e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kirill=20M=C3=BCller?= Date: Sun, 5 Jul 2020 16:19:35 +0200 Subject: [PATCH 7/7] Auto union --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 1067ba83f..5f9fae8a5 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ /NAMESPACE merge=union +/.Rbuildignore merge=union