updates to compare

Flank · Dec 14, 2020 · a5e7cbb · a5e7cbb
1 parent d59cca8
commit a5e7cbb
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 1 deletion.
diff --git a/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare b/integration_tests/src/test/resources/compare/MultipleDevicesIT-android-compare
@@ -1 +1 @@
-AndroidArgs    gcloud:      results-bucket: test-lab-[a-zA-Z0-9-]*      results-dir: [.a-zA-Z0-9_-]*      record-video: false      timeout: 15m      async: false      client-details:      network-profile: null      results-history-name: null      # Android gcloud      app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk      test: null      additional-apks:      auto-google-login: false      use-orchestrator: false      directories-to-pull:      grant-permissions: all      type: null      other-files:      scenario-numbers:      scenario-labels:      obb-files:      obb-names:      performance-metrics: false      num-uniform-shards: null      test-runner-class: null      test-targets:      robo-directives:      robo-script: null      device:        - model: NexusLowRes          version: 28          locale: en          orientation: portrait        - model: Pixel2          version: 28          locale: en          orientation: portrait        - model: HUR          version: 28          locale: en          orientation: portrait      num-flaky-test-attempts: 2      test-targets-for-shard:      fail-fast: false\s*    flank:      max-test-shards: 5      shard-time: -1      num-test-runs: 1      smart-flank-gcs-path:\s      smart-flank-disable-upload: false      default-test-time: 120.0      use-average-test-time-for-new-tests: false      files-to-download:      test-targets-always-run:      disable-sharding: false      project: flank-open-source      local-result-dir: results      full-junit-result: false      # Android Flank Yml      keep-file-path: false      additional-app-test-apks:        - app: null          test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk        - app: null          test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk        - app: null          test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk      run-timeout: -1      legacy-junit-result: false      ignore-failed-tests: false      output-style: single      disable-results-upload: false      default-class-test-time: 240.0\s*RunTests\s* Smart Flank cache hit: 0\% \(0 \/ 9\)  Shard times: 240s, 240s, 360s, 360s, 360s\s*\s* Smart Flank cache hit: 0\% \(0 \/ 9\)  Shard times: 240s, 240s, 360s, 360s, 360s\s*\s* Smart Flank cache hit: 0\% \(0 \/ 1\)  Shard times: 120s\s*Saved 3 shards to android_shards.json  Uploading android_shards.json \.*  Uploading app-debug.apk \.*[\s\S]*  11 tests \+ 8 parameterized classes \/ 11 shards\s*  3 matrix ids created in \d{1,2}m \d{1,2}s  https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\s*Matrices webLink  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?[\s\S]*CostReport  Physical devices    \$\d{1,2}.\d{1,2} for \d{1,2}m\s*  Virtual devices    \$\d{1,2}.\d{1,2} for \d{1,2}m\s*  Total    \$\d{1,2}.\d{1,2} for \d{1,2}m\s*  Uploading CostReport.txt \.*MatrixResultsReport  2 \/ 3 \(66\.67\%\)  1 matrices failed[\s\S]*More details are available at:https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?\s*  Uploading MatrixResultsReport.txt \.*  Uploading HtmlErrorReport.html \.*  Uploading JUnitReport.xml \.*[\s\S]*FetchArtifacts  \.*  Updating matrix file\s*Matrices webLink  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?
+AndroidArgs    gcloud:      results-bucket: test-lab-[a-zA-Z0-9-]*      results-dir: [.a-zA-Z0-9_-]*      record-video: false      timeout: 15m      async: false      client-details:      network-profile: null      results-history-name: null      # Android gcloud      app: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk      test: null      additional-apks:      auto-google-login: false      use-orchestrator: false      directories-to-pull:      grant-permissions: all      type: null      other-files:      scenario-numbers:      scenario-labels:      obb-files:      obb-names:      performance-metrics: false      num-uniform-shards: null      test-runner-class: null      test-targets:      robo-directives:      robo-script: null      device:        - model: NexusLowRes          version: 28          locale: en          orientation: portrait        - model: Pixel2          version: 28          locale: en          orientation: portrait        - model: HUR          version: 28          locale: en          orientation: portrait      num-flaky-test-attempts: 2      test-targets-for-shard:      fail-fast: false\s*    flank:      max-test-shards: 5      shard-time: -1      num-test-runs: 1      smart-flank-gcs-path:\s      smart-flank-disable-upload: false      default-test-time: 120.0      use-average-test-time-for-new-tests: false      files-to-download:      test-targets-always-run:      disable-sharding: false      project: flank-open-source      local-result-dir: results      full-junit-result: false      # Android Flank Yml      keep-file-path: false      additional-app-test-apks:        - app: null          test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk        - app: null          test: [0-9a-zA-Z\\\/_.:-]*[\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk        - app: null          test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk      run-timeout: -1      legacy-junit-result: false      ignore-failed-tests: false      output-style: single      disable-results-upload: false      default-class-test-time: 240.0\s*RunTests\s* Smart Flank cache hit: 0\% \(0 \/ 9\)  Shard times: 240s, 240s, 360s, 360s, 360s\s*\s* Smart Flank cache hit: 0\% \(0 \/ 9\)  Shard times: 240s, 240s, 360s, 360s, 360s\s*\s* Smart Flank cache hit: 0\% \(0 \/ 1\)  Shard times: 120s\s*Saved 3 shards to android_shards.json  Uploading android_shards.json \.*  Uploading app-debug.apk \.*[\s\S]*  11 tests \+ 8 parameterized classes \/ 11 shards\s*  3 matrix ids created in \d{1,2}m \d{1,2}s  https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]*\/[.a-zA-Z0-9_-]*\s*Matrices webLink  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?[\s\S]*CostReport  Physical devices    \$\d{1,2}.\d{1,2} for \d{1,2}m\s*  Virtual devices    \$\d{1,2}.\d{1,2} for \d{1,2}m\s*  Total    \$\d{1,2}.\d{1,2} for \d{1,2}m\s*  Uploading CostReport.txt \.*MatrixResultsReport  2 \/ 3 \(66\.67\%\)  1 matrices failed[\s\S]*More details are available at:https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?\s*  Uploading MatrixResultsReport.txt \.*  Uploading HtmlErrorReport.html \.*  Uploading JUnitReport.xml \.*  Uploading matrix_ids.json \.*[\s\S]*FetchArtifacts  \.*  Updating matrix file\s*Matrices webLink  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?  matrix-[a-zA-Z0-9]* https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]*\/matrices\/[.a-zA-Z0-9_-]*(\/executions\/[.a-zA-Z0-9_-]*)?

diff --git a/integration_tests/src/test/resources/compare/SanityRoboIT-compare b/integration_tests/src/test/resources/compare/SanityRoboIT-compare
@@ -85,6 +85,7 @@ MatrixResultsReport
 .*
   Uploading MatrixResultsReport.txt \.*
   Uploading JUnitReport.xml \.*
+  Uploading matrix_ids.json \.*
 FetchArtifacts
   \.*
   Updating matrix file

diff --git a/integration_tests/src/test/resources/compare/TestFilteringIT-compare b/integration_tests/src/test/resources/compare/TestFilteringIT-compare
@@ -89,6 +89,7 @@ MatrixResultsReport
 [\s\S]*
   Uploading MatrixResultsReport.txt \.*
   Uploading JUnitReport.xml \.*
+  Uploading matrix_ids.json \.*
 FetchArtifacts
   \.*
   Updating matrix file
Original file line number	Diff line number	Diff line change
		@@ -1 +1 @@
		AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-][\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false\s flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-][\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-][\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0\sRunTests\s Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s\s\s Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s\s\s Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s\sSaved 3 shards to android_shards.json Uploading android_shards.json \. Uploading app-debug.apk \.[\s\S] 11 tests \+ 8 parameterized classes \/ 11 shards\s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]\/[.a-zA-Z0-9_-]\sMatrices webLink matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])?[\s\S]CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m\s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m\s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m\s* Uploading CostReport.txt \.MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed[\s\S]More details are available at:https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])?\s Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.[\s\S]FetchArtifacts \.* Updating matrix file\sMatrices webLink matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-]*)?
		AndroidArgs gcloud: results-bucket: test-lab-[a-zA-Z0-9-]* results-dir: [.a-zA-Z0-9_-]* record-video: false timeout: 15m async: false client-details: network-profile: null results-history-name: null # Android gcloud app: [0-9a-zA-Z\\\/_.:-][\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-debug.apk test: null additional-apks: auto-google-login: false use-orchestrator: false directories-to-pull: grant-permissions: all type: null other-files: scenario-numbers: scenario-labels: obb-files: obb-names: performance-metrics: false num-uniform-shards: null test-runner-class: null test-targets: robo-directives: robo-script: null device: - model: NexusLowRes version: 28 locale: en orientation: portrait - model: Pixel2 version: 28 locale: en orientation: portrait - model: HUR version: 28 locale: en orientation: portrait num-flaky-test-attempts: 2 test-targets-for-shard: fail-fast: false\s flank: max-test-shards: 5 shard-time: -1 num-test-runs: 1 smart-flank-gcs-path:\s smart-flank-disable-upload: false default-test-time: 120.0 use-average-test-time-for-new-tests: false files-to-download: test-targets-always-run: disable-sharding: false project: flank-open-source local-result-dir: results full-junit-result: false # Android Flank Yml keep-file-path: false additional-app-test-apks: - app: null test: [0-9a-zA-Z\\\/_.:-][\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-success-debug-androidTest.apk - app: null test: [0-9a-zA-Z\\\/_.:-][\\\/]test_runner[\\\/]src[\\\/]test[\\\/]kotlin[\\\/]ftl[\\\/]fixtures[\\\/]tmp[\\\/]apk[\\\/]app-multiple-error-debug-androidTest.apk - app: null test: gs:\/\/flank-open-source.appspot.com\/integration\/app-single-success-debug-androidTest.apk run-timeout: -1 legacy-junit-result: false ignore-failed-tests: false output-style: single disable-results-upload: false default-class-test-time: 240.0\sRunTests\s Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s\s\s Smart Flank cache hit: 0\% \(0 \/ 9\) Shard times: 240s, 240s, 360s, 360s, 360s\s\s Smart Flank cache hit: 0\% \(0 \/ 1\) Shard times: 120s\sSaved 3 shards to android_shards.json Uploading android_shards.json \. Uploading app-debug.apk \.[\s\S] 11 tests \+ 8 parameterized classes \/ 11 shards\s* 3 matrix ids created in \d{1,2}m \d{1,2}s https:\/\/console.developers.google.com\/storage\/browser\/test-lab-[a-zA-Z0-9-]\/[.a-zA-Z0-9_-]\sMatrices webLink matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])?[\s\S]CostReport Physical devices \$\d{1,2}.\d{1,2} for \d{1,2}m\s* Virtual devices \$\d{1,2}.\d{1,2} for \d{1,2}m\s* Total \$\d{1,2}.\d{1,2} for \d{1,2}m\s* Uploading CostReport.txt \.MatrixResultsReport 2 \/ 3 \(66\.67\%\) 1 matrices failed[\s\S]More details are available at:https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])?\s Uploading MatrixResultsReport.txt \.* Uploading HtmlErrorReport.html \.* Uploading JUnitReport.xml \.* Uploading matrix_ids.json \.[\s\S]FetchArtifacts \.* Updating matrix file\sMatrices webLink matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-])? matrix-[a-zA-Z0-9] https:\/\/console.firebase.google.com\/project\/flank-open-source\/testlab\/histories\/[.a-zA-Z0-9_-]\/matrices\/[.a-zA-Z0-9_-](\/executions\/[.a-zA-Z0-9_-]*)?
Expand Down