Merge cockroachdb#106463

106463: interactive_tests: preserve demo logs in tests r=knz a=rafiss Now we pass in the --log-dir option so that logs are saved when the test fails. Otherwise, the demo command does not log. This also unskips two tests that should be working now. informs cockroachdb#96450 informs cockroachdb#102257 informs cockroachdb#100319 informs cockroachdb#106462 informs cockroachdb#106461 informs cockroachdb#96797 informs cockroachdb#96239 Release note: None Co-authored-by: Rafi Shamim <[email protected]>
xinhaoz · Jul 8, 2023 · 43c26ae · 43c26ae
2 parents 96a5952 + 87371af
commit 43c26ae
Show file tree

Hide file tree

Showing 20 changed files with 63 additions and 49 deletions.
diff --git a/pkg/acceptance/generated_cli_test.go b/pkg/acceptance/generated_cli_test.go
diff --git a/.../interactive_tests/test_demo.tcl.disabled → pkg/cli/interactive_tests/test_demo.tcl b/.../interactive_tests/test_demo.tcl.disabled → pkg/cli/interactive_tests/test_demo.tcl
@@ -3,7 +3,7 @@
 source [file join [file dirname $argv0] common.tcl]
 
 start_test "Check that demo insecure says hello properly, multitenant"
-spawn $argv demo --no-line-editor --insecure=true --multitenant=true
+spawn $argv demo --no-line-editor --insecure=true --multitenant=true --log-dir=logs
 # Be polite.
 eexpect "Welcome"
 # Warn the user that they won't get persistence.
@@ -28,7 +28,7 @@ eexpect eof
 end_test
 
 start_test "Check that demo insecure says hello properly, singletenant"
-spawn $argv demo --no-line-editor --insecure=true --multitenant=false
+spawn $argv demo --no-line-editor --insecure=true --multitenant=false --log-dir=logs
 # Check that the connection URL is printed; and that we're still
 # using default ports.
 eexpect "(webui)"
@@ -44,7 +44,7 @@ end_test
 start_test "Check that demo insecure, env var, says hello properly"
 # With env var.
 set ::env(COCKROACH_INSECURE) "true"
-spawn $argv demo --no-line-editor --no-example-database
+spawn $argv demo --no-line-editor --no-example-database --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 end_test
@@ -79,7 +79,7 @@ eexpect eof
 
 # With command-line override.
 set ::env(COCKROACH_INSECURE) "false"
-spawn $argv demo --no-line-editor --insecure=true --no-example-database
+spawn $argv demo --no-line-editor --insecure=true --no-example-database --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 
@@ -103,7 +103,7 @@ start_test "Check that demo secure says hello properly"
 
 # With env var.
 set ::env(COCKROACH_INSECURE) "false"
-spawn $argv demo --no-line-editor --no-example-database
+spawn $argv demo --no-line-editor --no-example-database --log-dir=logs
 eexpect "Welcome"
 
 eexpect "(webui)"
@@ -144,7 +144,7 @@ eexpect eof
 
 # With command-line override.
 set ::env(COCKROACH_INSECURE) "true"
-spawn $argv demo --no-line-editor --insecure=false --no-example-database
+spawn $argv demo --no-line-editor --insecure=false --no-example-database --log-dir=logs
 eexpect "Welcome"
 eexpect "Username: \"demo\", password"
 eexpect "defaultdb>"
@@ -168,7 +168,7 @@ end_test
 
 start_test "Check that the user can override the password."
 set ::env(COCKROACH_DEMO_PASSWORD) "hunter2"
-spawn $argv demo --no-line-editor --insecure=false --no-example-database
+spawn $argv demo --no-line-editor --insecure=false --no-example-database --log-dir=logs
 eexpect "Connection parameters"
 eexpect "(sql)"
 eexpect "postgresql://demo:hunter2@"
@@ -179,7 +179,7 @@ end_test
 
 # Test that demo displays connection URLs for nodes in the cluster.
 start_test "Check that node URLs are displayed"
-spawn $argv demo --no-line-editor --insecure --no-example-database
+spawn $argv demo --no-line-editor --insecure --no-example-database --log-dir=logs
 # Check that we see our message.
 eexpect "Connection parameters"
 eexpect "(webui)"
@@ -189,7 +189,7 @@ send_eof
 eexpect eof
 
 # Start the test again with a multi node cluster.
-spawn $argv demo --no-line-editor --insecure --nodes 3 --no-example-database
+spawn $argv demo --no-line-editor --insecure --nodes 3 --no-example-database --log-dir=logs
 
 # Check that we get a message for each node.
 eexpect "Connection parameters"
@@ -215,7 +215,7 @@ eexpect "defaultdb>"
 send_eof
 eexpect eof
 
-spawn $argv demo --no-line-editor --insecure=false --no-example-database
+spawn $argv demo --no-line-editor --insecure=false --no-example-database --log-dir=logs
 # Expect that security related tags are part of the connection URL.
 eexpect "(sql)"
 eexpect "sslmode=require"
@@ -249,7 +249,7 @@ end_test
 
 start_test "Check that the port numbers can be overridden from the command line."
 
-spawn $argv demo --no-line-editor --no-example-database --nodes 3 --http-port 8000
+spawn $argv demo --no-line-editor --no-example-database --nodes 3 --http-port 8000 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 
@@ -266,7 +266,7 @@ eexpect "defaultdb>"
 send_eof
 eexpect eof
 
-spawn $argv demo --no-line-editor --no-example-database --nodes 3 --sql-port 23000
+spawn $argv demo --no-line-editor --no-example-database --nodes 3 --sql-port 23000 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 
@@ -314,7 +314,7 @@ end_test
 
 start_test "Check that demo populates the connection URL in a configured file"
 
-spawn $argv demo --no-line-editor --no-example-database --listening-url-file=test.url
+spawn $argv demo --no-line-editor --no-example-database --listening-url-file=test.url --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 
@@ -325,7 +325,7 @@ send_eof
 eexpect eof
 
 # Ditto, insecure
-spawn $argv demo --no-line-editor --no-example-database --listening-url-file=test.url --insecure
+spawn $argv demo --no-line-editor --no-example-database --listening-url-file=test.url --insecure --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 

diff --git a/pkg/cli/interactive_tests/test_demo_changefeeds.tcl b/pkg/cli/interactive_tests/test_demo_changefeeds.tcl
@@ -3,7 +3,7 @@
 source [file join [file dirname $argv0] common.tcl]
 
 start_test "Demo core changefeed using format=csv"
-spawn $argv demo --no-line-editor --format=csv
+spawn $argv demo --no-line-editor --format=csv --log-dir=logs
 
 # We should start in a populated database.
 eexpect "movr>"
@@ -24,7 +24,7 @@ eexpect eof
 end_test
 
 start_test "Demo with rangefeeds disabled as they are in real life"
-spawn $argv demo --no-line-editor --format=csv --auto-enable-rangefeeds=false
+spawn $argv demo --no-line-editor --format=csv --auto-enable-rangefeeds=false --log-dir=logs
 
 # We should start in a populated database.
 eexpect "movr>"

diff --git a/pkg/cli/interactive_tests/test_demo_cli_integration.tcl b/pkg/cli/interactive_tests/test_demo_cli_integration.tcl
@@ -5,7 +5,7 @@ source [file join [file dirname $argv0] common.tcl]
 set ::env(COCKROACH_INSECURE) "false"
 set python "python2.7"
 
-spawn $argv demo --empty --no-line-editor --multitenant=true
+spawn $argv demo --empty --no-line-editor --multitenant=true --log-dir=logs
 eexpect "Welcome"
 
 start_test "Check that the cli connect instructions get printed out."
@@ -183,7 +183,7 @@ set spawn_id $demo_spawn_id
 send "\\q\r"
 eexpect eof
 
-spawn $argv demo --empty --no-line-editor --multitenant=true
+spawn $argv demo --empty --no-line-editor --multitenant=true --log-dir=logs
 set demo_spawn_id $spawn_id
 eexpect "Welcome"
 eexpect "defaultdb>"
@@ -208,7 +208,7 @@ eexpect eof
 
 
 start_test "Check that the warning for the system tenant is not printed when there are no secondary tenants"
-spawn $argv demo --empty --no-line-editor --multitenant=false
+spawn $argv demo --empty --no-line-editor --multitenant=false --log-dir=logs
 eexpect "Welcome"
 expect {
     "ATTENTION: YOU ARE CONNECTED TO THE SYSTEM TENANT" { exit 1 }

diff --git a/pkg/cli/interactive_tests/test_demo_global.tcl b/pkg/cli/interactive_tests/test_demo_global.tcl
@@ -9,7 +9,7 @@ start_test "Check --global flag runs as expected"
 
 # Start a demo with --global set
 # TODO(ajstorm): Disable multitenancy until #76305 is resolved.
-spawn $argv demo --no-line-editor --no-example-database --nodes 9 --global --multitenant=false
+spawn $argv demo --no-line-editor --no-example-database --nodes 9 --global --multitenant=false --log-dir=logs
 
 # Ensure db is defaultdb.
 eexpect "defaultdb>"

diff --git a/pkg/cli/interactive_tests/test_demo_global_insecure.tcl b/pkg/cli/interactive_tests/test_demo_global_insecure.tcl
@@ -9,7 +9,7 @@ start_test "Check --global flag runs as expected"
 
 # Start a demo with --global set
 # TODO(ajstorm): Disable multitenancy until #76305 is resolved.
-spawn $argv demo --no-line-editor --no-example-database --nodes 9 --global --multitenant=false --insecure
+spawn $argv demo --no-line-editor --no-example-database --nodes 9 --global --multitenant=false --insecure --log-dir=logs
 
 # Ensure db is defaultdb.
 eexpect "defaultdb>"

diff --git a/pkg/cli/interactive_tests/test_demo_locality_error.tcl b/pkg/cli/interactive_tests/test_demo_locality_error.tcl
@@ -5,13 +5,13 @@ source [file join [file dirname $argv0] common.tcl]
 start_test "Expect error with incorrect demo locality settings"
 
 # Test failure with less localities than expected.
-spawn $argv demo --nodes 3 --demo-locality=region=us-east1:region=us-east2
+spawn $argv demo --nodes 3 --demo-locality=region=us-east1:region=us-east2 --log-dir=logs
 # wait for the CLI to start up
 eexpect "ERROR: number of localities specified must equal number of nodes"
 eexpect eof
 
 # Test failure with more localities than expected.
-spawn $argv demo --nodes 3 --demo-locality=region=us-east1:region=us-east2:region=us-east3:region=us-east4
+spawn $argv demo --nodes 3 --demo-locality=region=us-east1:region=us-east2:region=us-east3:region=us-east4 --log-dir=logs
 # wait for the CLI to start up
 eexpect "ERROR: number of localities specified must equal number of nodes"
 eexpect eof

diff --git a/pkg/cli/interactive_tests/test_demo_memory_warning.tcl b/pkg/cli/interactive_tests/test_demo_memory_warning.tcl
@@ -3,7 +3,7 @@
 source [file join [file dirname $argv0] common.tcl]
 
 start_test "Check demo alerts when there is a memory warning."
-spawn $argv demo --no-line-editor --no-example-database --max-sql-memory=10% --nodes=8
+spawn $argv demo --no-line-editor --no-example-database --max-sql-memory=10% --nodes=8 --log-dir=logs
 
 eexpect "WARNING: HIGH MEMORY USAGE"
 

diff --git a/pkg/cli/interactive_tests/test_demo_multitenant.tcl.disabled b/pkg/cli/interactive_tests/test_demo_multitenant.tcl.disabled
@@ -8,7 +8,7 @@ source [file join [file dirname $argv0] common.tcl]
 start_test "Check --multitenant flag runs as expected"
 
 # Start a demo with --multitenant set
-spawn $argv demo --no-line-editor --empty --nodes 3 --multitenant
+spawn $argv demo --no-line-editor --empty --nodes 3 --multitenant --log-dir=logs
 
 eexpect "Welcome"
 eexpect "You are connected to tenant \"demoapp\""

diff --git a/...e_tests/test_demo_networking.tcl.disabled → ...nteractive_tests/test_demo_networking.tcl b/...e_tests/test_demo_networking.tcl.disabled → ...nteractive_tests/test_demo_networking.tcl
@@ -5,7 +5,7 @@ source [file join [file dirname $argv0] common.tcl]
 set ::env(COCKROACH_INSECURE) "false"
 
 start_test "Check that default demo has fixed ports, no multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2
+spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -32,7 +32,7 @@ eexpect eof
 end_test
 
 start_test "Check that http port can be randomized, no multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --http-port 0
+spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --http-port 0 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -57,7 +57,7 @@ eexpect eof
 end_test
 
 start_test "Check that sql port can be randomized, no multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --sql-port 0
+spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --sql-port 0 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -80,7 +80,7 @@ eexpect eof
 end_test
 
 start_test "Check that sql and http port can be randomized, no multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --sql-port 0 --http-port 0
+spawn $argv demo --empty --no-line-editor --multitenant=false --nodes 2 --sql-port 0 --http-port 0 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -101,7 +101,7 @@ eexpect eof
 end_test
 
 start_test "Check that default demo has fixed ports, w/ multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2
+spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -140,7 +140,7 @@ eexpect eof
 end_test
 
 start_test "Check that http port can be randomized, w/ multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --http-port 0
+spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --http-port 0 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -177,7 +177,7 @@ eexpect eof
 end_test
 
 start_test "Check that sql port can be randomized, w/ multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --sql-port 0
+spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --sql-port 0 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"
@@ -208,7 +208,7 @@ eexpect eof
 end_test
 
 start_test "Check that sql and http port can be randomized, w/ multitenancy"
-spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --sql-port 0 --http-port 0
+spawn $argv demo --empty --no-line-editor --multitenant=true --nodes 2 --sql-port 0 --http-port 0 --log-dir=logs
 eexpect "Welcome"
 eexpect "defaultdb>"
 send "\\demo ls\r"

diff --git a/pkg/cli/interactive_tests/test_demo_node_cmds.tcl b/pkg/cli/interactive_tests/test_demo_node_cmds.tcl
@@ -5,7 +5,7 @@ source [file join [file dirname $argv0] common.tcl]
 start_test "Check \\demo commands work as expected"
 # Start a demo with 5 nodes. Set multitenant=false due to unsupported
 # gossip commands below.
-spawn $argv demo --empty --no-line-editor --nodes=5 --multitenant=false
+spawn $argv demo --empty --no-line-editor --nodes=5 --multitenant=false --log-dir=logs --log-dir=logs
 
 eexpect "defaultdb>"
 

diff --git a/pkg/cli/interactive_tests/test_demo_partitioning.tcl.disabled b/pkg/cli/interactive_tests/test_demo_partitioning.tcl.disabled
@@ -20,7 +20,7 @@ eexpect $prompt
 
 start_test "Expect partitioning succeeds"
 # test that partitioning works if a license could be acquired
-send "$argv demo --no-line-editor --multitenant=false --geo-partitioned-replicas\r"
+send "$argv demo --no-line-editor --multitenant=false --geo-partitioned-replicas --log-dir=logs \r"
 
 # wait for the shell to start up
 eexpect "Partitioning the demo database"
@@ -84,7 +84,7 @@ eexpect $prompt
 end_test
 
 start_test "test multi-region setup"
-send "$argv demo movr --no-line-editor --geo-partitioned-replicas --multi-region\r"
+send "$argv demo movr --no-line-editor --geo-partitioned-replicas --multi-region --log-dir=logs \r"
 eexpect "Partitioning the demo database"
 eexpect "movr>"
 
@@ -106,7 +106,7 @@ eexpect "movr>"
 send_eof
 eexpect $prompt
 
-send "$argv demo movr --no-line-editor --geo-partitioned-replicas --multi-region --survive=region\r"
+send "$argv demo movr --no-line-editor --geo-partitioned-replicas --multi-region --survive=region --log-dir=logs \r"
 eexpect "Partitioning the demo database"
 eexpect "movr>"
 
@@ -134,15 +134,15 @@ eexpect $prompt
 end_test
 
 start_test "Expect an error if geo-partitioning is requested with multitenant mode"
-send "$argv demo --no-line-editor --geo-partitioned-replicas\r"
+send "$argv demo --no-line-editor --geo-partitioned-replicas --log-dir=logs \r"
 # expect a failure
 eexpect "operation is unsupported in multi-tenancy mode"
 eexpect $prompt
 end_test
 
 start_test "Expect an error if geo-partitioning is requested and license acquisition is disabled"
 # set the proper environment variable
-send "$argv demo --no-line-editor --geo-partitioned-replicas --disable-demo-license\r"
+send "$argv demo --no-line-editor --geo-partitioned-replicas --disable-demo-license --log-dir=logs \r"
 # expect a failure
 eexpect "enterprise features are needed for this demo"
 eexpect $prompt

diff --git a/pkg/cli/interactive_tests/test_demo_telemetry.tcl b/pkg/cli/interactive_tests/test_demo_telemetry.tcl
@@ -6,7 +6,7 @@ start_test "Check cockroach demo telemetry can be disabled"
 
 # set the proper environment variable
 set env(COCKROACH_SKIP_ENABLING_DIAGNOSTIC_REPORTING) "true"
-spawn $argv demo --no-line-editor
+spawn $argv demo --no-line-editor --log-dir=logs
 
 # Expect an informational message.
 eexpect "Telemetry disabled by configuration."