Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

TestStaticHostUserHandler flakiness #51173

Open
rosstimothy opened this issue Jan 17, 2025 · 0 comments
Open

TestStaticHostUserHandler flakiness #51173

rosstimothy opened this issue Jan 17, 2025 · 0 comments
Assignees
@atburke
Labels
flaky tests

Comments

@rosstimothy
Copy link
Contributor

Failure

Link(s) to logs

Relevant snippet

=== FAIL: lib/srv  (0.00s)
-test.shuffle 1737123130236264080
panic: test timed out after 10m0s
	running tests:
		TestStaticHostUserHandler (9m34s)
		TestStaticHostUserHandler/restart_on_watcher_init_failure (9m34s)

goroutine 3136 [running]:
testing.(*M).startAlarm.func1()
	/opt/go/src/testing/testing.go:2373 +0x265
created by time.goFunc
	/opt/go/src/time/sleep.go:215 +0x45

goroutine 1 [chan receive, 9 minutes, locked to thread]:
testing.tRunner.func1()
	/opt/go/src/testing/testing.go:1651 +0x9b2
testing.tRunner(0xc0007ec4e0, 0xc00127fa98)
	/opt/go/src/testing/testing.go:1696 +0x265
testing.runTests(0xc00059e288, {0x14ad6d00, 0x45, 0x45}, {0x2?, 0x39cbf2c?, 0x14d27640?})
	/opt/go/src/testing/testing.go:2166 +0x8bf
testing.(*M).Run(0xc000a29c20)
	/opt/go/src/testing/testing.go:2034 +0xf18
github.com/gravitational/teleport/lib/srv.TestMain(0xc000a29c20)
	/__w/teleport/teleport/lib/srv/exec_test.go:52 +0x1fd
main.main()
	_testmain.go:193 +0x172

goroutine 5 [select]:
go.opencensus.io/stats/view.(*worker).start(0xc000426600)
	/go/pkg/mod/[email protected]/stats/view/worker.go:292 +0x1a5
created by go.opencensus.io/stats/view.init.0 in goroutine 1
	/go/pkg/mod/[email protected]/stats/view/worker.go:34 +0xfd

goroutine 13 [select, 10 minutes]:
github.com/gravitational/teleport/lib/services.(*SemaphoreLock).keepAlive(0xc0008ebe00, {0x1007b468, 0xc000afaff0})
	/__w/teleport/teleport/lib/services/semaphore.go:211 +0x3b3
created by github.com/gravitational/teleport/lib/services.AcquireSemaphoreLock in goroutine 12
	/__w/teleport/teleport/lib/services/semaphore.go:322 +0x765

goroutine 14 [chan receive, 10 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionController).EnforceConnectionLimits.func1()
	/__w/teleport/teleport/lib/srv/session_control.go:304 +0x7e
created by github.com/gravitational/teleport/lib/srv.(*SessionController).EnforceConnectionLimits in goroutine 12
	/__w/teleport/teleport/lib/srv/session_control.go:301 +0x6e5

goroutine 1221 [select, 9 minutes]:
io.(*pipe).read(0xc002500660, {0xc00210e400, 0x400, 0x39c3d09?})
	/opt/go/src/io/pipe.go:57 +0x149
io.(*PipeReader).Read(0xc002500660, {0xc00210e400, 0x400, 0x400})
	/opt/go/src/io/pipe.go:134 +0x47
github.com/gravitational/teleport/lib/srv.(*TermManager).AddReader.func1()
	/__w/teleport/teleport/lib/srv/termmanager.go:244 +0xed
created by github.com/gravitational/teleport/lib/srv.(*TermManager).AddReader in goroutine 291
	/__w/teleport/teleport/lib/srv/termmanager.go:241 +0x1e7

goroutine 830 [chan receive, 9 minutes]:
testing.(*T).Run(0xc00162a4e0, {0xf2ca10a, 0x1f}, 0xc001694828)
	/opt/go/src/testing/testing.go:1751 +0x851
github.com/gravitational/teleport/lib/srv.TestStaticHostUserHandler(0xc00162a4e0)
	/__w/teleport/teleport/lib/srv/statichostuser_test.go:380 +0x5e25
testing.tRunner(0xc00162a4e0, 0xf4aada0)
	/opt/go/src/testing/testing.go:1690 +0x227
created by testing.(*T).Run in goroutine 1
	/opt/go/src/testing/testing.go:1743 +0x826

goroutine 868 [select, 9 minutes]:
io.(*pipe).read(0xc00019a480, {0xc00210e800, 0x400, 0x39c3d09?})
	/opt/go/src/io/pipe.go:57 +0x149
io.(*PipeReader).Read(0xc00019a480, {0xc00210e800, 0x400, 0x400})
	/opt/go/src/io/pipe.go:134 +0x47
github.com/gravitational/teleport/lib/srv.(*TermManager).AddReader.func1()
	/__w/teleport/teleport/lib/srv/termmanager.go:244 +0xed
created by github.com/gravitational/teleport/lib/srv.(*TermManager).AddReader in goroutine 833
	/__w/teleport/teleport/lib/srv/termmanager.go:241 +0x1e7

goroutine 2555 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc001a5ede0, {0x1007b430, 0xc0015d0a80}, {0x7fb844dc3198, 0xc00242d080})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 2668
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 2405 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc0017154d0, {0x1007b430, 0xc00041a150}, {0x7fb844dc3198, 0xc001a071a0})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 2331
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 1751 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc00123f830, {0x1007b430, 0xc001e96120}, {0x7fb844dc3198, 0xc001719bc0})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 1490
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 1307 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*StaticHostUserHandler).Run(0xc00074ebd0, {0x1007b468, 0xc0007f8c30})
	/__w/teleport/teleport/lib/srv/statichostusers.go:125 +0x32b
github.com/gravitational/teleport/lib/srv.TestStaticHostUserHandler.func5(0xc00162b520)
	/__w/teleport/teleport/lib/srv/statichostuser_test.go:415 +0x91f
testing.tRunner(0xc00162b520, 0xc001694828)
	/opt/go/src/testing/testing.go:1690 +0x227
created by testing.(*T).Run in goroutine 830
	/opt/go/src/testing/testing.go:1743 +0x826

goroutine 634 [select, 9 minutes]:
io.(*pipe).read(0xc001718ae0, {0xc00[2190](https://github.com/gravitational/teleport/actions/runs/12830576724/job/35779083645?pr=50799#step:6:2191)000, 0x400, 0x39c3d09?})
	/opt/go/src/io/pipe.go:57 +0x149
io.(*PipeReader).Read(0xc001718ae0, {0xc002190000, 0x400, 0x400})
	/opt/go/src/io/pipe.go:134 +0x47
github.com/gravitational/teleport/lib/srv.(*TermManager).AddReader.func1()
	/__w/teleport/teleport/lib/srv/termmanager.go:244 +0xed
created by github.com/gravitational/teleport/lib/srv.(*TermManager).AddReader in goroutine 710
	/__w/teleport/teleport/lib/srv/termmanager.go:241 +0x1e7

goroutine 2430 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc001b61230, {0x1007b430, 0xc00271c0c0}, {0x7fb844dc3198, 0xc001926840})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 2374
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 1308 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.TestStaticHostUserHandler.func3.TestStaticHostUserHandler.func1.1({0x1007b468, 0xc0007f8c80}, 0xc0000e4740, {0x3?, 0x0?})
	/__w/teleport/teleport/lib/srv/statichostuser_test.go:137 +0x19a
github.com/gravitational/teleport/lib/srv.TestStaticHostUserHandler.func3({0x1007b468, 0xc0007f8c80}, 0xc0000e4740, {0x10098100, 0xc0017df4a0})
	/__w/teleport/teleport/lib/srv/statichostuser_test.go:328 +0x182b
github.com/gravitational/teleport/lib/srv.TestStaticHostUserHandler.func5.1({0x1007b468, 0xc0007f8c80})
	/__w/teleport/teleport/lib/srv/statichostuser_test.go:399 +0x86
github.com/gravitational/teleport/lib/utils.RunTestBackgroundTask.func1()
	/__w/teleport/teleport/lib/utils/testhelpers.go:61 +0xef
created by github.com/gravitational/teleport/lib/utils.RunTestBackgroundTask in goroutine 1307
	/__w/teleport/teleport/lib/utils/testhelpers.go:59 +0x18b

goroutine 1342 [chan receive, 9 minutes]:
github.com/gravitational/teleport/lib/srv.newSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:844 +0xe5
created by github.com/gravitational/teleport/lib/srv.newSession in goroutine 290
	/__w/teleport/teleport/lib/srv/sess.go:843 +0x15b9

goroutine 1345 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc001803110, {0x1007b430, 0xc001803140}, {0x7fb844dc3198, 0xc001971e60})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 290
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 2242 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc0017140f0, {0x1007b430, 0xc001714300}, {0x7fb844dc3198, 0xc002451140})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 1990
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 2373 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc000711710, {0x1007b430, 0xc0018bee10}, {0x7fb844dc3198, 0xc0023b5c80})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine [2244](https://github.com/gravitational/teleport/actions/runs/12830576724/job/35779083645?pr=50799#step:6:2245)
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a

goroutine 2667 [select, 9 minutes]:
github.com/gravitational/teleport/lib/srv.(*SessionTracker).UpdateExpirationLoop(0xc0022d35f0, {0x1007b430, 0xc0015d07e0}, {0x7fb844dc3198, 0xc001782c00})
	/__w/teleport/teleport/lib/srv/sessiontracker.go:93 +0x2ef
github.com/gravitational/teleport/lib/srv.(*session).trackSession.func1()
	/__w/teleport/teleport/lib/srv/sess.go:2337 +0x8ae
created by github.com/gravitational/teleport/lib/srv.(*session).trackSession in goroutine 2629
	/__w/teleport/teleport/lib/srv/sess.go:2325 +0x171a
FAIL	github.com/gravitational/teleport/lib/srv	600.261s

=== FAIL: lib/srv TestStaticHostUserHandler (unknown)
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/usermgmt.go:430","message":"Attempting to upsert host user","host_username":"bob","mode":2,"uid":"","gid":""}
{"caller":"srv/heartbeat.go:165","component":"test:beat","level":"debug","message":"Starting Auth heartbeat with announce period: 10s, keep-alive period 1m0s, poll period: 1s","timestamp":"2025-01-17T14:12:36Z"}
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/usermgmt.go:446","message":"Aborting host user creation, can't update unmanaged user unless explicitly migrating.","host_username":"alice","mode":3,"uid":"","gid":"","login":"alice"}
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/usermgmt.go:430","message":"Attempting to upsert host user","host_username":"alice","mode":3,"uid":"","gid":""}
    testhelpers.go:76: Cleanup: terminating test background task "event sender".
--- PASS: TestStaticHostUserHandler/ok_users (0.00s)
    testhelpers.go:76: Cleanup: terminating test background task "event sender".
--- PASS: TestStaticHostUserHandler/ignore_non-matching_user (0.00s)
{"caller":"srv/heartbeat.go:602","component":"test:beat","level":"warning","message":"Keep alive has failed: ooops.","timestamp":"2025-01-17T14:12:36Z"}
{"timestamp":"2025-01-17T14:12:36Z","level":"info","caller":"srv/usermgmt.go:371","message":"Creating host user","host_username":"bob","mode":2,"uid":"","shell":"","gid":""}
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"lite/lite.go:259","message":"Connected to database","component":"sqlite","database":"file:/tmp/TestTrackingSessionnode_with_proxy_recording_mode2983447845/001/sqlite.db?_busy_timeout=10000&_sync=FULL&_txlock=immediate","poll_stream_period":1000000000}
    testhelpers.go:76: Cleanup: terminating test background task "event sender".
--- PASS: TestStaticHostUserHandler/ignore_multiple_matches (0.00s)
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"lite/lite.go:259","message":"Connected to database","component":"sqlite","database":"file:/tmp/TestNonInteractiveSessionwith_BPF865[2266](https://github.com/gravitational/teleport/actions/runs/12830576724/job/35779083645?pr=50799#step:6:2267)39/001/sqlite.db?_busy_timeout=10000&_sync=FULL&_txlock=immediate","poll_stream_period":1000000000}
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/statichostusers.go:256","message":"Attempt to update matched static host user.","login":"test"}
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/statichostusers.go:256","message":"Attempt to update matched static host user.","login":"test"}
    testhelpers.go:76: Cleanup: terminating test background task "event sender".
--- PASS: TestStaticHostUserHandler/update_user (0.00s)
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/statichostusers.go:122","message":"Static host user handler encountered a network error, will restart.","error":"missing init event from watcher"}
{"timestamp":"2025-01-17T14:12:36Z","level":"debug","caller":"srv/statichostusers.go:122","message":"Static host user handler encountered a network error, will restart.","error":"timed out waiting for static host user watcher to initialize"}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Assignees

@atburke atburke

Labels
flaky tests
Projects
None yet
Milestone
No milestone
Development

No branches or pull requests
2 participants
@atburke @rosstimothy