runtime: flaky corruption on openbsd

[rsc]

#!watchflakes
post <- goos == "openbsd" && (`runtime.throw` || `Test ".*" ran over .* limit` || `panic: test timed out` || `fatal error: all goroutines are asleep`) && !(log ~ `unexpected fault address 0x349dc4`) && date < "2023-08-01"

OpenBSD has been seeing a steady stream of what looks like memory corruption resulting in runtime throws.

[gopherbot]

Found new matching flaky dashboard failures.

2022-08-09 21:26 openbsd-386-70 go@d39b5417 (log)

fatal error: workbuf is empty

runtime stack:
runtime.throw({0x86206fc, 0x10})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x458ba844 sp=0x458ba830 pc=0x807e8ad
runtime.(*workbuf).checknonempty(0x6ae88000)
	/tmp/workdir/go/src/runtime/mgcwork.go:337 +0x34 fp=0x458ba850 sp=0x458ba844 pc=0x806f764
runtime.trygetfull()
	/tmp/workdir/go/src/runtime/mgcwork.go:429 +0x31 fp=0x458ba860 sp=0x458ba850 pc=0x806fa91
runtime.(*gcWork).tryGet(0x6a83025c)
	/tmp/workdir/go/src/runtime/mgcwork.go:215 +0x4e fp=0x458ba874 sp=0x458ba860 pc=0x806f52e
runtime.gcDrain(0x6a83025c, 0x7)
	/tmp/workdir/go/src/runtime/mgcmark.go:1090 +0x1ec fp=0x458ba8d8 sp=0x458ba874 pc=0x8066bdc
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1326 +0x6b fp=0x458ba904 sp=0x458ba8d8 pc=0x806261b
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_386.s:370 +0x35 fp=0x458ba908 sp=0x458ba904 pc=0x80aaec5

2022-08-10 23:27 openbsd-amd64-68 go@29b07037 cmd/api (log)

2022/08/10 16:41:32 loading imports: exit status 2
runtime: want=4 got=0
fatal error: limiterEvent.stop: found wrong event in p's limiter event slot

runtime stack:
runtime.throw({0xa47d87?, 0x3?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x292bab418 sp=0x292bab3e8 pc=0x43625d
runtime.(*limiterEvent).stop(0x100000000000800?, 0x4, 0xb59cc6?)
	/tmp/workdir/go/src/runtime/mgclimit.go:455 +0x145 fp=0x292bab440 sp=0x292bab418 pc=0x41e385
runtime.pidleget(0xdf91e0?)
...
cmd/go/internal/work.(*Builder).build(0xc0001b8320, {0xb2dc10?, 0xc000026140?}, 0xc0004ce500)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:476 +0x199 fp=0xc000453db8 sp=0xc000453338 pc=0x8bf9d9
cmd/go/internal/work.(*Builder).Do.func2({0xb2dc10, 0xc000026140}, 0xc0004ce500)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:139 +0x5fc fp=0xc000453f20 sp=0xc000453db8 pc=0x8bd2fc
cmd/go/internal/work.(*Builder).Do.func3()
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:201 +0xb9 fp=0xc000453fe0 sp=0xc000453f20 pc=0x8bcb19
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000453fe8 sp=0xc000453fe0 pc=0x466941
created by cmd/go/internal/work.(*Builder).Do
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:187 +0x3f5

2022-08-11 20:13 openbsd-amd64-68 go@87431981 (log)

runtime: want=4 got=0
fatal error: limiterEvent.stop: found wrong event in p's limiter event slot

runtime stack:
runtime.throw({0xa47e87?, 0x3?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x28baa2e88 sp=0x28baa2e58 pc=0x43625d
runtime.(*limiterEvent).stop(0x100058800?, 0x4, 0xb59f00?)
	/tmp/workdir/go/src/runtime/mgclimit.go:455 +0x145 fp=0x28baa2eb0 sp=0x28baa2e88 pc=0x41e385
runtime.pidleget(0x28baa2f40?)
	/tmp/workdir/go/src/runtime/proc.go:5735 +0x130 fp=0x28baa2ee8 sp=0x28baa2eb0 pc=0x4451d0
...
cmd/go/internal/work.(*Builder).build(0xc000250c80, {0xc000026150?, 0x0?}, 0xc0003f9900)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:870 +0x2b79 fp=0xc000731db8 sp=0xc000731338 pc=0x8c2af9
cmd/go/internal/work.(*Builder).Do.func2({0xb2ddd0, 0xc000026150}, 0xc0003f9900)
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:139 +0x5fc fp=0xc000731f20 sp=0xc000731db8 pc=0x8bda3c
cmd/go/internal/work.(*Builder).Do.func3()
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:201 +0xb9 fp=0xc000731fe0 sp=0xc000731f20 pc=0x8bd259
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000731fe8 sp=0xc000731fe0 pc=0x466941
created by cmd/go/internal/work.(*Builder).Do
	/tmp/workdir/go/src/cmd/go/internal/work/exec.go:187 +0x3f5

2022-08-26 14:32 openbsd-amd64-70 go@d7a3fa12 (log)

fatal error: workbuf is empty
fatal error: workbuf is empty
fatal error: workbuf is empty
fatal error: workbuf is empty

runtime stack:
runtime.throw({0xa2cbb7?, 0x0?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x7f7ffffe5f80 sp=0x7f7ffffe5f50 pc=0x434a7d
runtime.(*workbuf).checknonempty(0x0?)
	/tmp/workdir/go/src/runtime/mgcwork.go:338 +0x36 fp=0x7f7ffffe5fa0 sp=0x7f7ffffe5f80 pc=0x426376
...
runtime.(*gcWork).tryGet(0xc000047738)
	/tmp/workdir/go/src/runtime/mgcwork.go:216 +0x5b fp=0x2555ad4e8 sp=0x2555ad4b8 pc=0x4260fb
runtime.gcDrain(0xc000047738, 0x7)
	/tmp/workdir/go/src/runtime/mgcmark.go:1092 +0x185 fp=0x2555ad548 sp=0x2555ad4e8 pc=0x41f3e5
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1326 +0x6d fp=0x2555ad598 sp=0x2555ad548 pc=0x41bb0d
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x2555ad5a0 sp=0x2555ad598 pc=0x4640c9

go tool dist: FAILED: go list -f={{if .Stale}}	STALE {{.ImportPath}}: {{.StaleReason}}{{end}} std cmd: exit status 2

2022-09-02 06:06 openbsd-amd64-68 go@134cd34c (log)

wirep: p->m=0(0) p->status=2
fatal error: wirep: invalid p state
runtime: want=4 got=0
fatal error: limiterEvent.stop: found wrong event in p's limiter event slot

runtime stack:
runtime.throw({0xa2cfc5?, 0x21f41c8d0?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x21f41c890 sp=0x21f41c860 pc=0x434a3d
runtime.wirep(0xc000502820?)
	/tmp/workdir/go/src/runtime/proc.go:5028 +0x105 fp=0x21f41c8c8 sp=0x21f41c890 pc=0x441be5
...
runtime.pidleget(0x465e24?)
	/tmp/workdir/go/src/runtime/proc.go:5806 +0x131 fp=0x2886a0f18 sp=0x2886a0ee0 pc=0x443911
runtime.exitsyscallfast_pidle()
	/tmp/workdir/go/src/runtime/proc.go:3971 +0x2c fp=0x2886a0f38 sp=0x2886a0f18 pc=0x43f42c
runtime.exitsyscallfast.func1()
	/tmp/workdir/go/src/runtime/proc.go:3927 +0x38 fp=0x2886a0f68 sp=0x2886a0f38 pc=0x43f298
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x2886a0f70 sp=0x2886a0f68 pc=0x463f69

go tool dist: FAILED: go list -f={{if .Stale}}	STALE {{.ImportPath}}: {{.StaleReason}}{{end}} std cmd: exit status 2

2022-09-06 18:16 openbsd-386-70 go@ef69718d (log)

runtime: pointer 0x64002680 to unused region of span span.base()=0x642d8000 span.limit=0x642e0000 span.state=1
runtime: found in object at *(0x64239340+0x1fc)
object=0x64239340 s.base()=0x64238000 s.limit=0x64239e40 s.spanclass=58 s.elemsize=704 s.state=mSpanInUse
 *(object+0) = 0x63d808a0
 *(object+4) = 0x1b
 *(object+8) = 0x63c9e2d4
 *(object+12) = 0x7
 *(object+16) = 0x0
 *(object+20) = 0x0
 *(object+24) = 0x31bf3cca
...
fatal error: found bad pointer in Go heap (incorrect use of unsafe or cgo?)

runtime stack:
runtime.throw({0x8644df0, 0x3e})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x128b8688 sp=0x128b8674 pc=0x807d85d
runtime.badPointer(0x3a99c820, 0x64002680, 0x64239340, 0x1fc)
	/tmp/workdir/go/src/runtime/mbitmap.go:314 +0x96 fp=0x128b86a8 sp=0x128b8688 pc=0x805ab76
runtime.findObject(0x64002680, 0x64239340, 0x1fc)
	/tmp/workdir/go/src/runtime/mbitmap.go:357 +0x9f fp=0x128b86c0 sp=0x128b86a8 pc=0x805adcf
runtime.scanobject(0x64239340, 0x63c2ef5c)
...
runtime.gcDrainN(0x63c2ef5c, 0x10000)
	/tmp/workdir/go/src/runtime/mgcmark.go:1194 +0x20b fp=0x128b8734 sp=0x128b8710 pc=0x806652b
runtime.gcAssistAlloc1(0x63c07b30, 0x10000)
	/tmp/workdir/go/src/runtime/mgcmark.go:569 +0x12e fp=0x128b8794 sp=0x128b8734 pc=0x8064bbe
runtime.gcAssistAlloc.func1()
	/tmp/workdir/go/src/runtime/mgcmark.go:476 +0x28 fp=0x128b87a4 sp=0x128b8794 pc=0x8064a78
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_386.s:370 +0x35 fp=0x128b87a8 sp=0x128b87a4 pc=0x80ab095

go tool dist: FAILED: go list -f={{if .Stale}}	STALE {{.ImportPath}}: {{.StaleReason}}{{end}} std: exit status 2

2022-09-07 18:41 openbsd-386-70 go@e828fbdf (log)

fatal error: workbuf is not empty
fatal error: workbuf is not empty

runtime stack:
runtime.throw({0x862b06b, 0x14})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x341aefac sp=0x341aef98 pc=0x807d85d
runtime.(*workbuf).checkempty(0x67ebd000)
	/tmp/workdir/go/src/runtime/mgcwork.go:344 +0x34 fp=0x341aefb8 sp=0x341aefac pc=0x806e8a4
runtime.getempty()
	/tmp/workdir/go/src/runtime/mgcwork.go:357 +0x55 fp=0x341aefe4 sp=0x341aefb8 pc=0x806e905
...
	/tmp/workdir/go/src/runtime/mgcwork.go:439 +0x14 fp=0x22356cc8 sp=0x22356cb4 pc=0x806ebc4
runtime.(*gcWork).balance(0x67836f5c)
	/tmp/workdir/go/src/runtime/mgcwork.go:297 +0x31 fp=0x22356cd4 sp=0x22356cc8 pc=0x806e7d1
runtime.gcDrain(0x67836f5c, 0x7)
	/tmp/workdir/go/src/runtime/mgcmark.go:1087 +0x1a2 fp=0x22356d38 sp=0x22356cd4 pc=0x8065f62
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1326 +0x6b fp=0x22356d64 sp=0x22356d38 pc=0x8061a5b
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_386.s:370 +0x35 fp=0x22356d68 sp=0x22356d64 pc=0x80ab0f5
go tool dist: Failed: exit status 2

2022-09-07 20:10 openbsd-amd64-68 go@454a058f (log)

XXXBANNERXXX:Test execution environment.
# GOARCH: amd64
# CPU: Intel(R) Xeon(R) CPU @ 2.80GHz
# GOOS: openbsd
# OS Version: OpenBSD 6.8 OpenBSD 6.8 (GENERIC.MP) #2: Sat Dec  5 07:17:48 MST 2020     root@syspatch-68-amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP  amd64
runtime: marked free object in span 0x2cd1e1c20, elemsize=32 freeindex=7 (bad use of unsafe.Pointer? try -d=checkptr)
0xc0000fa000 alloc marked  
0xc0000fa020 alloc marked  
0xc0000fa040 alloc marked  
0xc0000fa060 alloc marked  
...
fatal error: found pointer to free object

runtime stack:
runtime.throw({0xa302d3?, 0xc0000fb660?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x2c1f0e598 sp=0x2c1f0e568 pc=0x434a5d
runtime.(*mspan).reportZombies(0x2cd1e1c20)
	/tmp/workdir/go/src/runtime/mgcsweep.go:788 +0x2e5 fp=0x2c1f0e618 sp=0x2c1f0e598 pc=0x425b05
runtime.(*sweepLocked).sweep(0xc000358338?, 0x0)
	/tmp/workdir/go/src/runtime/mgcsweep.go:612 +0x8f6 fp=0x2c1f0e6f8 sp=0x2c1f0e618 pc=0x425516
runtime.(*mcentral).uncacheSpan(0x2d76d3060?, 0xc0006da340?)
...
runtime.gcMarkDone()
	/tmp/workdir/go/src/runtime/mgc.go:918 +0x2aa fp=0xc000052f50 sp=0xc000052ef0 pc=0x41a82a
runtime.gcBgMarkWorker()
	/tmp/workdir/go/src/runtime/mgc.go:1366 +0x327 fp=0xc000052fe0 sp=0xc000052f50 pc=0x41b8e7
runtime.goexit()
	/tmp/workdir/go/src/runtime/asm_amd64.s:1594 +0x1 fp=0xc000052fe8 sp=0xc000052fe0 pc=0x466261
created by runtime.gcBgMarkStartWorkers
	/tmp/workdir/go/src/runtime/mgc.go:1158 +0x25

go tool dist: FAILED: go list -f={{if .Stale}}	STALE {{.ImportPath}}: {{.StaleReason}}{{end}} std: exit status 2

2022-09-08 14:57 openbsd-amd64-68 go@530a2369 (log)

fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x43a482]

runtime stack:
runtime.throw({0xa3be47?, 0xc0004031e0?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x22c11b5a0 sp=0x22c11b570 pc=0x434a5d
runtime.sigpanic()
	/tmp/workdir/go/src/runtime/signal_unix.go:821 +0x369 fp=0x22c11b5f8 sp=0x22c11b5a0 pc=0x44a609
runtime.runSafePointFn()
	/tmp/workdir/go/src/runtime/proc.go:1683 +0x42 fp=0x22c11b610 sp=0x22c11b5f8 pc=0x43a482
runtime.findRunnable()
	/tmp/workdir/go/src/runtime/proc.go:2590 +0x6b fp=0x22c11b718 sp=0x22c11b610 pc=0x43c06b
runtime.schedule()
	/tmp/workdir/go/src/runtime/proc.go:3264 +0xb1 fp=0x22c11b750 sp=0x22c11b718 pc=0x43d8d1
runtime.park_m(0xc000302ea0?)
	/tmp/workdir/go/src/runtime/proc.go:3413 +0x12d fp=0x22c11b780 sp=0x22c11b750 pc=0x43ddcd
runtime.mcall()
	/tmp/workdir/go/src/runtime/asm_amd64.s:448 +0x43 fp=0x22c11b790 sp=0x22c11b780 pc=0x464003

2022-09-14 16:11 openbsd-amd64-68 go@4baa4869 (log)

XXXBANNERXXX:Test execution environment.
# GOARCH: amd64
# CPU: Intel(R) Xeon(R) CPU @ 2.80GHz
# GOOS: openbsd
# OS Version: OpenBSD 6.8 OpenBSD 6.8 (GENERIC.MP) #2: Sat Dec  5 07:17:48 MST 2020     root@syspatch-68-amd64.openbsd.org:/usr/src/sys/arch/amd64/compile/GENERIC.MP  amd64
runtime: want=4 got=0
fatal error: limiterEvent.stop: found wrong event in p's limiter event slot

runtime stack:
runtime.throw({0xa44ec5?, 0x3?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x217806990 sp=0x217806960 pc=0x434a5d
runtime.(*limiterEvent).stop(0x0?, 0x4, 0x1?)
	/tmp/workdir/go/src/runtime/mgclimit.go:453 +0x145 fp=0x2178069b8 sp=0x217806990 pc=0x41cee5
runtime.pidleget(0xc000050f10?)
	/tmp/workdir/go/src/runtime/proc.go:5806 +0x131 fp=0x2178069f0 sp=0x2178069b8 pc=0x4439b1
runtime.pidlegetSpinning(0xc000028120?)
	/tmp/workdir/go/src/runtime/proc.go:5824 +0x19 fp=0x217806a08 sp=0x2178069f0 pc=0x443a39
runtime.wakep()
	/tmp/workdir/go/src/runtime/proc.go:2430 +0x70 fp=0x217806a38 sp=0x217806a08 pc=0x43b970
runtime.resetspinning()
	/tmp/workdir/go/src/runtime/proc.go:3148 +0x49 fp=0x217806a58 sp=0x217806a38 pc=0x43d509
runtime.schedule()
	/tmp/workdir/go/src/runtime/proc.go:3270 +0xd1 fp=0x217806a90 sp=0x217806a58 pc=0x43d8f1
runtime.park_m(0xc000082b60?)
	/tmp/workdir/go/src/runtime/proc.go:3413 +0x12d fp=0x217806ac0 sp=0x217806a90 pc=0x43ddcd
runtime.mcall()
	/tmp/workdir/go/src/runtime/asm_amd64.s:448 +0x43 fp=0x217806ad0 sp=0x217806ac0 pc=0x463fe3

go tool dist: FAILED: go list -f={{if .Stale}}	STALE {{.ImportPath}}: {{.StaleReason}}{{end}} std: exit status 2

2022-09-15 21:05 openbsd-amd64-70 go@629c0b3a (log)

fatal error: workbuf is empty
fatal error: workbuf is empty
fatal error: workbuf is empty
fatal error: workbuf is empty
fatal error: workbuf is empty

runtime stack:
runtime.throw({0xa26d67?, 0x298d9225?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x2141449d8 sp=0x2141449a8 pc=0x434a3d
runtime.(*workbuf).checknonempty(0x3?)
...
	/tmp/workdir/go/src/runtime/mgcwork.go:430 +0x65 fp=0x2490de568 sp=0x2490de548 pc=0x426745
runtime.(*gcWork).tryGet(0xc00003bc38)
	/tmp/workdir/go/src/runtime/mgcwork.go:216 +0x5b fp=0x2490de598 sp=0x2490de568 pc=0x4260bb
runtime.gcDrain(0xc00003bc38, 0x7)
	/tmp/workdir/go/src/runtime/mgcmark.go:1092 +0x185 fp=0x2490de5f8 sp=0x2490de598 pc=0x41f385
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1327 +0x6d fp=0x2490de648 sp=0x2490de5f8 pc=0x41ba8d
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x2490de650 sp=0x2490de648 pc=0x464049
go tool dist: Failed: exit status 2

2022-09-16 17:59 openbsd-amd64-68 go@a84f46a2 (log)

fatal error: forEachP: P did not run fn

runtime stack:
runtime.throw({0xa2c7c3?, 0xc0001fa1a0?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x280401998 sp=0x280401968 pc=0x434b7d
runtime.forEachP(0xa856a0)
	/tmp/workdir/go/src/runtime/proc.go:1710 +0x2fe fp=0x280401a00 sp=0x280401998 pc=0x43a5be
runtime.gcMarkDone.func1()
	/tmp/workdir/go/src/runtime/mgc.go:814 +0x47 fp=0x280401a28 sp=0x280401a00 pc=0x45f1a7
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x280401a30 sp=0x280401a28 pc=0x4642e9

— watchflakes

[gopherbot]

Sorry, but there were parse errors in the watchflakes script. The script I found was:

#!watchflakes
post <- goos == "openbsd" && `runtime.throw` || `Test ".*" ran over .* limit` ||

And the problems were:

script:3.1: unexpected end of script

— watchflakes

[gopherbot]

Found new matching flaky dashboard failures for:

#!watchflakes
post <- goos == "openbsd" && (`runtime.throw` || `Test ".*" ran over .* limit`)

2022-08-23 18:13 openbsd-amd64-68 go@72a76ca1 (log)

Test "go_test:cmd/compile/internal/ir" ran over 20m0s limit (20m0.000456651s); saw output:

XXXBANNERXXX:Test execution environment.

2022-09-16 20:39 openbsd-amd64-68 go@2d89bec2 (log)

Test "go_test:encoding/pem" ran over 20m0s limit (20m0.000756227s); saw output:

XXXBANNERXXX:Test execution environment.

— watchflakes

[gopherbot]

Found new matching flaky dashboard failures for:

#!watchflakes
post <- goos == "openbsd" && (`runtime.throw` || `Test ".*" ran over .* limit`)

2022-09-20 02:27 openbsd-386-70 debug@00090926 go@225bcec9 (log)

fatal error: workbuf is not empty

runtime stack:
runtime.throw({0x862473a, 0x14})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x4d fp=0x3594ad5c sp=0x3594ad48 pc=0x807e64d
runtime.(*workbuf).checkempty(0x708fb800)
	/tmp/workdir/go/src/runtime/mgcwork.go:343 +0x34 fp=0x3594ad68 sp=0x3594ad5c pc=0x806f544
runtime.getempty()
	/tmp/workdir/go/src/runtime/mgcwork.go:356 +0x55 fp=0x3594ad94 sp=0x3594ad68 pc=0x806f5a5
runtime.handoff(0x708fb800)
	/tmp/workdir/go/src/runtime/mgcwork.go:438 +0x14 fp=0x3594ada8 sp=0x3594ad94 pc=0x806f864
runtime.(*gcWork).balance(0x70034f5c)
	/tmp/workdir/go/src/runtime/mgcwork.go:296 +0x31 fp=0x3594adb4 sp=0x3594ada8 pc=0x806f471
runtime.gcDrain(0x70034f5c, 0x7)
	/tmp/workdir/go/src/runtime/mgcmark.go:1085 +0x1a1 fp=0x3594ae18 sp=0x3594adb4 pc=0x8066aa1
runtime.gcBgMarkWorker.func2()
	/tmp/workdir/go/src/runtime/mgc.go:1327 +0x6b fp=0x3594ae44 sp=0x3594ae18 pc=0x806253b
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_386.s:370 +0x35 fp=0x3594ae48 sp=0x3594ae44 pc=0x80aacd5

— watchflakes

[rsc]

Updated pattern to match panic test timed out.

[gopherbot]

Found new dashboard test flakes for:

#!watchflakes
post <- goos == "openbsd" && (`runtime.throw` || `Test ".*" ran over .* limit` || `panic: test timed out`)

2022-08-04 18:51 openbsd-386-70 tools@99fd76f9 go@39728f41 x/tools/gopls/internal/regtest/misc.TestFailingDiagnosticClearingOnEdit (log)

serve.go:442: debug server listening at http://localhost:41121
serve.go:442: debug server listening at http://localhost:28768
#### Start Gopls Test Logs for "TestFailingDiagnosticClearingOnEdit/default"
[Trace - 12:57:49.544 PM] Sending request 'initialize - (1)'.
Params: {"processId":0,"clientInfo":{"name":"fakeclient","version":"v1.0.0"},"rootUri":"","capabilities":{"workspace":{"didChangeConfiguration":{},"didChangeWatchedFiles":{"dynamicRegistration":true},"executeCommand":{},"semanticTokens":{},"codeLens":{},"inlineValue":{},"inlayHint":{},"configuration":true},"textDocument":{"synchronization":{},"completion":{"completionItem":{"snippetSupport":true,"tagSupport":{"valueSet":[1]},"resolveSupport":{"properties":null},"insertTextModeSupport":{"valueSet":null}},"completionItemKind":{},"completionList":{}},"hover":{},"signatureHelp":{"signatureInformation":{"parameterInformation":{}}},"declaration":{},"definition":{},"typeDefinition":{},"implementation":{},"references":{},"documentHighlight":{},"documentSymbol":{"symbolKind":{},"tagSupport":{"valueSet":null}},"codeAction":{"codeActionLiteralSupport":{"codeActionKind":{"valueSet":null}},"resolveSupport":{"properties":null}},"codeLens":{},"documentLink":{},"colorProvider":{},"formatting":{},"rangeFormatting":{},"onTypeFormatting":{},"rename":{},"foldingRange":{},"selectionRange":{},"publishDiagnostics":{"tagSupport":{"valueSet":null}},"callHierarchy":{},"semanticTokens":{"requests":{"full":true},"tokenTypes":["namespace","type","class","enum","interface","struct","typeParameter","parameter","variable","property","enumMember","event","function","method","macro","keyword","modifier","comment","string","number","regexp","operator"],"tokenModifiers":null,"formats":null},"linkedEditingRange":{},"moniker":{},"typeHierarchy":{},"inlineValue":{},"inlayHint":{"resolveSupport":{"properties":null}}},"window":{"workDoneProgress":true,"showMessage":{"messageActionItem":{}},"showDocument":{"support":false}},"general":{"staleRequestSupport":{"cancel":false,"retryOnContentModified":null},"regularExpressions":{"engine":""},"markdown":{"parser":""}}},"initializationOptions":{"completionBudget":"10s","diagnosticsDelay":"10ms","env":{"GO111MODULE":"","GOMODCACHE":"","GOPACKAGESDRIVER":"off","GOPATH":"/tmp/workdir/tmp/gopls-regtest-1768495668/TestFailingDiagnosticClearingOnEdit/default/gopath","GOPROXY":"file:///tmp/workdir/tmp/gopls-regtest-1768495668/TestFailingDiagnosticClearingOnEdit/default/proxy","GOSUMDB":"off"},"verboseWorkDoneProgress":true},"trace":"messages","workspaceFolders":[{"uri":"file:///tmp/workdir/tmp/gopls-regtest-1768495668/TestFailingDiagnosticClearingOnEdit/default/work","name":"work"}]}


[Trace - 12:57:49.546 PM] Received response 'initialize - (1)' in 1ms.
Result: {"capabilities":{"textDocumentSync":{"openClose":true,"change":2,"save":{}},"completionProvider":{"triggerCharacters":["."],"completionItem":{}},"hoverProvider":true,"signatureHelpProvider":{"triggerCharacters":["(",","]},"definitionProvider":true,"typeDefinitionProvider":true,"implementationProvider":true,"referencesProvider":true,"documentHighlightProvider":true,"documentSymbolProvider":true,"codeActionProvider":true,"codeLensProvider":{},"documentLinkProvider":{},"workspaceSymbolProvider":true,"documentFormattingProvider":true,"documentOnTypeFormattingProvider":{"firstTriggerCharacter":""},"renameProvider":true,"foldingRangeProvider":true,"executeCommandProvider":{"commands":["gopls.add_dependency","gopls.add_import","gopls.apply_fix","gopls.check_upgrades","gopls.edit_go_directive","gopls.gc_details","gopls.generate","gopls.generate_gopls_mod","gopls.go_get_package","gopls.list_imports","gopls.list_known_packages","gopls.regenerate_cgo","gopls.remove_dependency","gopls.run_tests","gopls.run_vulncheck_exp","gopls.start_debugging","gopls.test","gopls.tidy","gopls.toggle_gc_details","gopls.update_go_sum","gopls.upgrade_dependency","gopls.vendor"]},"callHierarchyProvider":true,"workspace":{"workspaceFolders":{"supported":true,"changeNotifications":"workspace/didChangeWorkspaceFolders"}},"inlayHintProvider":{}},"serverInfo":{"name":"gopls","version":"{\"GoVersion\":\"devel 39728f412d5fb6d97568cc84a42f1caf07dbaedc\",\"Path\":\"\",\"Main\":{\"Path\":\"\",\"Version\":\"\",\"Sum\":\"\",\"Replace\":null},\"Deps\":null,\"Settings\":null,\"Version\":\"master\"}"}}

...
--- FAIL: TestFailingDiagnosticClearingOnEdit (464.32s)
    --- FAIL: TestFailingDiagnosticClearingOnEdit/default (464.32s)
        env.go:279: waiting on:
            Met: no diagnostics for "a.go"
            Unmet: no diagnostics for "b.go"

            err:context deadline exceeded

            state:
            #### log messages (see RPC logs for full text):
...


#### End Gopls Test Logs for "TestCRLFLineEndings/47200/default"
panic: test timed out after 10m0s

goroutine 18591 [running]:
testing.(*M).startAlarm.func1()
	/tmp/workdir/go/src/testing/testing.go:2036 +0xc5
created by time.goFunc
	/tmp/workdir/go/src/time/sleep.go:176 +0x29

2022-08-08 18:11 openbsd-386-70 tools@92d58ea4 go@0981d9ff x/tools/gopls/internal/regtest/misc.TestFailingDiagnosticClearingOnEdit (log)

serve.go:442: debug server listening at http://localhost:26925
serve.go:442: debug server listening at http://localhost:44022
#### Start Gopls Test Logs for "TestFailingDiagnosticClearingOnEdit/default"
[Trace - 07:53:17.205 AM] Sending request 'initialize - (1)'.
Params: {"processId":0,"clientInfo":{"name":"fakeclient","version":"v1.0.0"},"rootUri":"","capabilities":{"workspace":{"didChangeConfiguration":{},"didChangeWatchedFiles":{"dynamicRegistration":true},"executeCommand":{},"semanticTokens":{},"codeLens":{},"inlineValue":{},"inlayHint":{},"configuration":true},"textDocument":{"synchronization":{},"completion":{"completionItem":{"snippetSupport":true,"tagSupport":{"valueSet":[1]},"resolveSupport":{"properties":null},"insertTextModeSupport":{"valueSet":null}},"completionItemKind":{},"completionList":{}},"hover":{},"signatureHelp":{"signatureInformation":{"parameterInformation":{}}},"declaration":{},"definition":{},"typeDefinition":{},"implementation":{},"references":{},"documentHighlight":{},"documentSymbol":{"symbolKind":{},"tagSupport":{"valueSet":null}},"codeAction":{"codeActionLiteralSupport":{"codeActionKind":{"valueSet":null}},"resolveSupport":{"properties":null}},"codeLens":{},"documentLink":{},"colorProvider":{},"formatting":{},"rangeFormatting":{},"onTypeFormatting":{},"rename":{},"foldingRange":{},"selectionRange":{},"publishDiagnostics":{"tagSupport":{"valueSet":null}},"callHierarchy":{},"semanticTokens":{"requests":{"full":true},"tokenTypes":["namespace","type","class","enum","interface","struct","typeParameter","parameter","variable","property","enumMember","event","function","method","macro","keyword","modifier","comment","string","number","regexp","operator"],"tokenModifiers":null,"formats":null},"linkedEditingRange":{},"moniker":{},"typeHierarchy":{},"inlineValue":{},"inlayHint":{"resolveSupport":{"properties":null}}},"window":{"workDoneProgress":true,"showMessage":{"messageActionItem":{}},"showDocument":{"support":false}},"general":{"staleRequestSupport":{"cancel":false,"retryOnContentModified":null},"regularExpressions":{"engine":""},"markdown":{"parser":""}}},"initializationOptions":{"completionBudget":"10s","diagnosticsDelay":"10ms","env":{"GO111MODULE":"","GOMODCACHE":"","GOPACKAGESDRIVER":"off","GOPATH":"/tmp/workdir/tmp/gopls-regtest-1009992577/TestFailingDiagnosticClearingOnEdit/default/gopath","GOPROXY":"file:///tmp/workdir/tmp/gopls-regtest-1009992577/TestFailingDiagnosticClearingOnEdit/default/proxy","GOSUMDB":"off"},"verboseWorkDoneProgress":true},"trace":"messages","workspaceFolders":[{"uri":"file:///tmp/workdir/tmp/gopls-regtest-1009992577/TestFailingDiagnosticClearingOnEdit/default/work","name":"work"}]}


[Trace - 07:53:17.210 AM] Received response 'initialize - (1)' in 4ms.
Result: {"capabilities":{"textDocumentSync":{"openClose":true,"change":2,"save":{}},"completionProvider":{"triggerCharacters":["."],"completionItem":{}},"hoverProvider":true,"signatureHelpProvider":{"triggerCharacters":["(",","]},"definitionProvider":true,"typeDefinitionProvider":true,"implementationProvider":true,"referencesProvider":true,"documentHighlightProvider":true,"documentSymbolProvider":true,"codeActionProvider":true,"codeLensProvider":{},"documentLinkProvider":{},"workspaceSymbolProvider":true,"documentFormattingProvider":true,"documentOnTypeFormattingProvider":{"firstTriggerCharacter":""},"renameProvider":true,"foldingRangeProvider":true,"executeCommandProvider":{"commands":["gopls.add_dependency","gopls.add_import","gopls.apply_fix","gopls.check_upgrades","gopls.edit_go_directive","gopls.gc_details","gopls.generate","gopls.generate_gopls_mod","gopls.go_get_package","gopls.list_imports","gopls.list_known_packages","gopls.regenerate_cgo","gopls.remove_dependency","gopls.run_tests","gopls.run_vulncheck_exp","gopls.start_debugging","gopls.test","gopls.tidy","gopls.toggle_gc_details","gopls.update_go_sum","gopls.upgrade_dependency","gopls.vendor"]},"callHierarchyProvider":true,"workspace":{"workspaceFolders":{"supported":true,"changeNotifications":"workspace/didChangeWorkspaceFolders"}},"inlayHintProvider":{}},"serverInfo":{"name":"gopls","version":"{\"GoVersion\":\"devel 0981d9fff1fa5601a2b7833473955252425bd923\",\"Path\":\"\",\"Main\":{\"Path\":\"\",\"Version\":\"\",\"Sum\":\"\",\"Replace\":null},\"Deps\":null,\"Settings\":null,\"Version\":\"master\"}"}}

...
--- FAIL: TestFailingDiagnosticClearingOnEdit (467.22s)
    --- FAIL: TestFailingDiagnosticClearingOnEdit/default (467.22s)
        env.go:286: waiting on:
            Met: no diagnostics for "a.go"
            Unmet: no diagnostics for "b.go"

            err:context deadline exceeded

            state:
            #### log messages (see RPC logs for full text):
...


#### End Gopls Test Logs for "TestCRLFLineEndings/47200/default"
panic: test timed out after 10m0s

goroutine 19174 [running]:
testing.(*M).startAlarm.func1()
	/tmp/workdir/go/src/testing/testing.go:2036 +0xc5
created by time.goFunc
	/tmp/workdir/go/src/time/sleep.go:176 +0x29

2022-09-26 16:36 openbsd-amd64-68 go@c3728b75 (log)

PASS





XXXBANNERXXX:Test execution environment.
# GOARCH: amd64
# CPU: Intel(R) Xeon(R) CPU @ 2.80GHz
# GOOS: openbsd
...
fatal error: wirep: invalid p state
runtime: want=4 got=0
fatal error: limiterEvent.stop: found wrong event in p's limiter event slot

runtime stack:
runtime.throw({0xa2acb1?, 0x2f7628418?})
	/tmp/workdir/go/src/runtime/panic.go:1047 +0x5d fp=0x2f76283d8 sp=0x2f76283a8 pc=0x434b3d
runtime.wirep(0xc00046c680?)
	/tmp/workdir/go/src/runtime/proc.go:5085 +0x105 fp=0x2f7628410 sp=0x2f76283d8 pc=0x441e05
runtime.acquirep(0xc000036500)
...
runtime.pidleget(0x466224?)
	/tmp/workdir/go/src/runtime/proc.go:5863 +0x131 fp=0x26f0f1818 sp=0x26f0f17e0 pc=0x443b31
runtime.exitsyscallfast_pidle()
	/tmp/workdir/go/src/runtime/proc.go:4028 +0x2c fp=0x26f0f1838 sp=0x26f0f1818 pc=0x43f62c
runtime.exitsyscallfast.func1()
	/tmp/workdir/go/src/runtime/proc.go:3984 +0x38 fp=0x26f0f1868 sp=0x26f0f1838 pc=0x43f498
runtime.systemstack()
	/tmp/workdir/go/src/runtime/asm_amd64.s:492 +0x49 fp=0x26f0f1870 sp=0x26f0f1868 pc=0x464369

go tool dist: FAILED: go list -f={{if .Stale}}	STALE {{.ImportPath}}: {{.StaleReason}}{{end}} std: exit status 2

2022-09-26 22:21 openbsd-386-68 tools@eb45e986 go@e6d9057e command-line-arguments.TestGolden [build] (log)

/tmp/workdir/tmp/stringer3302685186/day_string.go:11:8: invalid argument: index 1 out of bounds [0:1]
/tmp/workdir/tmp/stringer3302685186/day_string.go:12:8: invalid argument: index 1 out of bounds [0:1]
/tmp/workdir/tmp/stringer3302685186/day_string.go:13:8: invalid argument: index 1 out of bounds [0:1]
/tmp/workdir/tmp/stringer3302685186/day_string.go:14:8: invalid argument: index 1 out of bounds [0:1]
/tmp/workdir/tmp/stringer3302685186/day_string.go:15:8: invalid argument: index 1 out of bounds [0:1]
/tmp/workdir/tmp/stringer3302685186/day_string.go:16:8: invalid argument: index 1 out of bounds [0:1]
/tmp/workdir/tmp/stringer3302685186/day_string.go:17:8: invalid argument: index Sunday - 6 (constant -6 of type Day) must not be negative
panic: test timed out after 10m0s

golang.org/x/tools/internal/gocommand.runCmdContext({0x8323128, 0x6bea7e00}, 0x6beaa4d0)
...
	/tmp/workdir/gopath/src/golang.org/x/tools/go/packages/golist.go:199 +0x787
golang.org/x/tools/go/packages.defaultDriver(0x6beac604, {0x6be8b1d0, 0x1, 0x1})
	/tmp/workdir/gopath/src/golang.org/x/tools/go/packages/packages.go:271 +0x52
golang.org/x/tools/go/packages.Load(0x6bedde24, {0x6be8b1d0, 0x1, 0x1})
	/tmp/workdir/gopath/src/golang.org/x/tools/go/packages/packages.go:254 +0x4c
golang.org/x/tools/cmd/stringer.(*Generator).parsePackage(0x6bea7da0, {0x6be8b1d0, 0x1, 0x1}, {0x0, 0x0, 0x0})
	/tmp/workdir/gopath/src/golang.org/x/tools/cmd/stringer/stringer.go:226 +0x150
golang.org/x/tools/cmd/stringer.TestGolden(0x6bc823c0)
	/tmp/workdir/gopath/src/golang.org/x/tools/cmd/stringer/golden_test.go:474 +0x47f
testing.tRunner(0x6bc823c0, 0x82e7828)

2022-09-26 22:21 openbsd-386-68 tools@eb45e986 go@e6d9057e x/tools/go/packages.TestAll (log)

panic: test timed out after 10m0s

testing.(*T).Run(0x7443e960, {0x82ed5b8, 0x5}, 0x72c10540)
	/tmp/workdir/go/src/testing/testing.go:1529 +0x392
golang.org/x/tools/go/packages_test.testInvalidFilesBeforeOverlayContains(0x7443e960, {0x834578c, 0x84c373c})
	/tmp/workdir/gopath/src/golang.org/x/tools/go/packages/overlay_test.go:815 +0x8f4
golang.org/x/tools/go/packages_test.testAllOrModulesParallel.func1(0x7443e960, {0x834578c, 0x84c373c})
	/tmp/workdir/gopath/src/golang.org/x/tools/go/packages/packages_test.go:81 +0x185
golang.org/x/tools/go/packages/packagestest.TestAll.func1(0x7443e960)
	/tmp/workdir/gopath/src/golang.org/x/tools/go/packages/packagestest/export.go:163 +0x51
testing.tRunner(0x7443e960, 0x72982440)

— watchflakes

[mknyszek]

Because #34988 is still open these memory corruption issues are difficult to detangle from these being real runtime issues (vs. fork corruption).

@golang/openbsd Can you please take a look at these and let us know what you think?